From 2a0c81a12c7e6c5ac1557b0f1f4a581f23fd4ca7 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 24 Sep 2012 17:55:53 +0300 Subject: [PATCH] Add support for include_dir in config file. This allows easily splitting configuration into many files, deployed in a directory. Magnus Hagander, Greg Smith, Selena Deckelmann, reviewed by Noah Misch. --- doc/src/sgml/config.sgml | 150 +++++++++++--- src/backend/utils/misc/guc-file.l | 188 +++++++++++++++--- src/backend/utils/misc/postgresql.conf.sample | 11 + src/include/utils/guc.h | 5 + 4 files changed, 291 insertions(+), 63 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index cfdc803056..4bd06ed760 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -79,38 +79,6 @@ shared_buffers = 128MB value, write either two quotes (preferred) or backslash-quote. - - - include - in configuration file - - In addition to parameter settings, the postgresql.conf - file can contain include directives, which specify - another file to read and process as if it were inserted into the - configuration file at this point. This feature allows a configuration - file to be divided into physically separate parts. - Include directives simply look like: - -include 'filename' - - If the file name is not an absolute path, it is taken as relative to - the directory containing the referencing configuration file. - Inclusions can be nested. - - - - - include_if_exists - in configuration file - - There is also an include_if_exists directive, which acts - the same as the include directive, except for the behavior - when the referenced file does not exist or cannot be read. A regular - include will consider this an error condition, but - include_if_exists merely logs a message and continues - processing the referencing configuration file. - - SIGHUP @@ -213,7 +181,123 @@ SET ENABLE_SEQSCAN TO OFF; - + + + Configuration File Includes + + + + include + in configuration file + + In addition to parameter settings, the postgresql.conf + file can contain include directives, which specify + another file to read and process as if it were inserted into the + configuration file at this point. This feature allows a configuration + file to be divided into physically separate parts. + Include directives simply look like: + +include 'filename' + + If the file name is not an absolute path, it is taken as relative to + the directory containing the referencing configuration file. + Inclusions can be nested. + + + + + include_if_exists + in configuration file + + There is also an include_if_exists directive, which acts + the same as the include directive, except for the behavior + when the referenced file does not exist or cannot be read. A regular + include will consider this an error condition, but + include_if_exists merely logs a message and continues + processing the referencing configuration file. + + + + + include_dir + in configuration file + + The postgresql.conf file can also contain + include_dir directives, which specify an entire directory + of configuration files to include. It is used similarly: + + include_dir 'directory' + + Non-absolute directory names follow the same rules as single file include + directives: they are relative to the directory containing the referencing + configuration file. Within that directory, only non-directory files whose + names end with the suffix .conf will be included. File + names that start with the . character are also excluded, + to prevent mistakes as they are hidden on some platforms. Multiple files + within an include directory are processed in filename order. The filenames + are ordered by C locale rules, ie. numbers before letters, and uppercase + letters before lowercase ones. + + + + Include files or directories can be used to logically separate portions + of the database configuration, rather than having a single large + postgresql.conf file. Consider a company that has two + database servers, each with a different amount of memory. There are likely + elements of the configuration both will share, for things such as logging. + But memory-related parameters on the server will vary between the two. And + there might be server specific customizations, too. One way to manage this + situation is to break the custom configuration changes for your site into + three files. You could add this to the end of your + postgresql.conf file to include them: + + include 'shared.conf' + include 'memory.conf' + include 'server.conf' + + All systems would have the same shared.conf. Each server + with a particular amount of memory could share the same + memory.conf; you might have one for all servers with 8GB of RAM, + another for those having 16GB. And finally server.conf could + have truly server-specific configuration information in it. + + + + Another possibility is to create a configuration file directory and + put this information into files there. For example, a conf.d + directory could be referenced at the end ofpostgresql.conf: + + include_dir 'conf.d' + + Then you could name the files in the conf.d directory like this: + + 00shared.conf + 01memory.conf + 02server.conf + + This shows a clear order in which these files will be loaded. This is + important because only the last setting encountered when the server is + reading its configuration will be used. Something set in + conf.d/02server.conf in this example would override a value + set in conf.d/01memory.conf. + + + + You might instead use this configuration directory approach while naming + these files more descriptively: + + 00shared.conf + 01memory-8GB.conf + 02server-foo.conf + + This sort of arrangement gives a unique name for each configuration file + variation. This can help eliminate ambiguity when several servers have + their configurations all stored in one place, such as in a version + control repository. (Storing database configuration files under version + control is another good practice to consider). + + + File Locations diff --git a/src/backend/utils/misc/guc-file.l b/src/backend/utils/misc/guc-file.l index ca7619034f..52d540e4cd 100644 --- a/src/backend/utils/misc/guc-file.l +++ b/src/backend/utils/misc/guc-file.l @@ -362,6 +362,39 @@ ProcessConfigFile(GucContext context) } } +/* + * Given a configuration file or directory location that may be a relative + * path, return an absolute one. We consider the location to be relative to + * the directory holding the calling file. + */ +static char * +AbsoluteConfigLocation(const char *location, const char *calling_file) +{ + char abs_path[MAXPGPATH]; + + if (is_absolute_path(location)) + return pstrdup(location); + else + { + if (calling_file != NULL) + { + strlcpy(abs_path, calling_file, sizeof(abs_path)); + get_parent_directory(abs_path); + join_path_components(abs_path, abs_path, location); + canonicalize_path(abs_path); + } + else + { + /* + * calling_file is NULL, we make an absolute path from $PGDATA + */ + join_path_components(abs_path, data_directory, location); + canonicalize_path(abs_path); + } + return pstrdup(abs_path); + } +} + /* * Read and parse a single configuration file. This function recurses * to handle "include" directives. @@ -378,7 +411,6 @@ ParseConfigFile(const char *config_file, const char *calling_file, bool strict, { bool OK = true; FILE *fp; - char abs_path[MAXPGPATH]; /* * Reject too-deep include nesting depth. This is just a safety check @@ -394,31 +426,7 @@ ParseConfigFile(const char *config_file, const char *calling_file, bool strict, return false; } - /* - * If config_file is a relative path, convert to absolute. We consider - * it to be relative to the directory holding the calling file. - */ - if (!is_absolute_path(config_file)) - { - if (calling_file != NULL) - { - strlcpy(abs_path, calling_file, sizeof(abs_path)); - get_parent_directory(abs_path); - join_path_components(abs_path, abs_path, config_file); - canonicalize_path(abs_path); - config_file = abs_path; - } - else - { - /* - * calling_file is NULL, we make an absolute path from $PGDATA - */ - join_path_components(abs_path, data_directory, config_file); - canonicalize_path(abs_path); - config_file = abs_path; - } - } - + config_file = AbsoluteConfigLocation(config_file,calling_file); fp = AllocateFile(config_file, "r"); if (!fp) { @@ -563,7 +571,22 @@ ParseConfigFp(FILE *fp, const char *config_file, int depth, int elevel, } /* OK, process the option name and value */ - if (guc_name_compare(opt_name, "include_if_exists") == 0) + if (guc_name_compare(opt_name, "include_dir") == 0) + { + /* + * An include_dir directive isn't a variable and should be + * processed immediately. + */ + if (!ParseConfigDirectory(opt_value, config_file, + depth + 1, elevel, + head_p, tail_p)) + OK = false; + yy_switch_to_buffer(lex_buffer); + ConfigFileLineno = save_ConfigFileLineno; + pfree(opt_name); + pfree(opt_value); + } + else if (guc_name_compare(opt_name, "include_if_exists") == 0) { /* * An include_if_exists directive isn't a variable and should be @@ -573,9 +596,9 @@ ParseConfigFp(FILE *fp, const char *config_file, int depth, int elevel, depth + 1, elevel, head_p, tail_p)) OK = false; - yy_switch_to_buffer(lex_buffer); - pfree(opt_name); - pfree(opt_value); + yy_switch_to_buffer(lex_buffer); + pfree(opt_name); + pfree(opt_value); } else if (guc_name_compare(opt_name, "include") == 0) { @@ -665,6 +688,111 @@ cleanup: return OK; } +/* + * Read and parse all config files in a subdirectory in alphabetical order + */ +bool +ParseConfigDirectory(const char *includedir, + const char *calling_file, + int depth, int elevel, + ConfigVariable **head_p, + ConfigVariable **tail_p) +{ + char *directory; + DIR *d; + struct dirent *de; + char **filenames = NULL; + int num_filenames = 0; + int size_filenames = 0; + bool status; + + directory = AbsoluteConfigLocation(includedir, calling_file); + d = AllocateDir(directory); + if (d == NULL) + { + ereport(elevel, + (errcode_for_file_access(), + errmsg("could not open configuration directory \"%s\": %m", + directory))); + return false; + } + + /* + * Read the directory and put the filenames in an array, so we can sort + * them prior to processing the contents. + */ + while ((de = ReadDir(d, directory)) != NULL) + { + struct stat st; + char filename[MAXPGPATH]; + + /* + * Only parse files with names ending in ".conf". Explicitly reject + * files starting with ".". This excludes things like "." and "..", + * as well as typical hidden files, backup files, and editor debris. + */ + if (strlen(de->d_name) < 6) + continue; + if (de->d_name[0] == '.') + continue; + if (strcmp(de->d_name + strlen(de->d_name) - 5, ".conf") != 0) + continue; + + join_path_components(filename, directory, de->d_name); + canonicalize_path(filename); + if (stat(filename, &st) == 0) + { + if (!S_ISDIR(st.st_mode)) + { + /* Add file to list, increasing its size in blocks of 32 */ + if (num_filenames == size_filenames) + { + size_filenames += 32; + if (num_filenames == 0) + /* Must initialize, repalloc won't take NULL input */ + filenames = palloc(size_filenames * sizeof(char *)); + else + filenames = repalloc(filenames, size_filenames * sizeof(char *)); + } + filenames[num_filenames] = pstrdup(filename); + num_filenames++; + } + } + else + { + /* + * stat does not care about permissions, so the most likely reason + * a file can't be accessed now is if it was removed between the + * directory listing and now. + */ + ereport(elevel, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", + filename))); + return false; + } + } + + if (num_filenames > 0) + { + int i; + qsort(filenames, num_filenames, sizeof(char *), pg_qsort_strcmp); + for (i = 0; i < num_filenames; i++) + { + if (!ParseConfigFile(filenames[i], NULL, true, + depth, elevel, head_p, tail_p)) + { + status = false; + goto cleanup; + } + } + } + status = true; + +cleanup: + FreeDir(d); + return status; +} /* * Free a list of ConfigVariables, including the names and the values diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index adcbcf6620..10f3fb1b24 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -566,6 +566,17 @@ #exit_on_error = off # terminate session on any error? #restart_after_crash = on # reinitialize after backend crash? +#------------------------------------------------------------------------------ +# CONFIG FILE INCLUDES +#------------------------------------------------------------------------------ + +# These options allow settings to be loaded from files other than the +# default postgresql.conf + +#include_dir = 'conf.d' # include files ending in '.conf' from + # directory 'conf.d' +#include_if_exists = 'exists.conf' # include file only if it exists +#include = 'special.conf' # include file #------------------------------------------------------------------------------ # CUSTOMIZED OPTIONS diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index 6810387755..06f797cb0a 100644 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -116,6 +116,11 @@ extern bool ParseConfigFile(const char *config_file, const char *calling_file, extern bool ParseConfigFp(FILE *fp, const char *config_file, int depth, int elevel, ConfigVariable **head_p, ConfigVariable **tail_p); +extern bool ParseConfigDirectory(const char *includedir, + const char *calling_file, + int depth, int elevel, + ConfigVariable **head_p, + ConfigVariable **tail_p); extern void FreeConfigVariables(ConfigVariable *list); /*