Add contrib/environment-to-ini (#9519)

* Add contrib/environment-to-ini This contrib command provides a mechanism to allow arbitrary setting of ini values using the environment variable in a more docker standard fashion. Environment variable keys should be structured as: "GITEA__SECTION_NAME__KEY_NAME" Use of the command is explained in the README. Partial fix for #350 Closes #7287 * Update contrib/environment-to-ini/environment-to-ini.go Co-Authored-By: 6543 <6543@obermui.de> Co-authored-by: Antoine GIRARD <sapk@users.noreply.github.com> Co-authored-by: 6543 <6543@obermui.de>
5 years ago · 884173232f
parent 4ee97465e9
commit 884173232f
2 changed files with 290 additions and 0 deletions
--- a/contrib/environment-to-ini/README
+++ b/contrib/environment-to-ini/README
@ -0,0 +1,66 @@
+Environment To Ini
+==================
+
+Multiple docker users have requested that the Gitea docker is changed
+to permit arbitrary configuration via environment variables.
+
+Gitea needs to use an ini file for configuration because the running
+environment that starts the docker may not be the same as that used
+by the hooks. An ini file also gives a good default and means that
+users do not have to completely provide a full environment.
+
+With those caveats above, this command provides a generic way of
+converting suitably structured environment variables into any ini
+value.
+
+To use the command is very simple just run it and the default gitea
+app.ini will be rewritten to take account of the variables provided,
+however there are various options to give slightly different
+behavior and these can be interrogated with the `-h` option.
+
+The environment variables should be of the form:
+
+	GITEA__SECTION_NAME__KEY_NAME
+
+Environment variables are usually restricted to a reduced character
+set "0-9A-Z_" - in order to allow the setting of sections with
+characters outside of that set, they should be escaped as following:
+"_0X2E_" for ".". The entire section and key names can be escaped as
+a UTF8 byte string if necessary. E.g. to configure:
+
+	"""
+	...
+	[log.console]
+	COLORIZE=false
+	STDERR=true
+	...
+	"""
+
+You would set the environment variables: "GITEA__LOG_0x2E_CONSOLE__COLORIZE=false"
+and "GITEA__LOG_0x2E_CONSOLE__STDERR=false". Other examples can be found
+on the configuration cheat sheet.
+
+To plug this command in to the docker, you simply compile the provided go file using:
+
+	go build environment-to-ini.go
+
+And copy the resulting `environment-to-ini` command to /app/gitea in the docker.
+
+Apply the below patch to /etc/s6/gitea.setup to wire this in.
+
+If you find this useful please comment on #7287
+
+
+diff --git a/docker/root/etc/s6/gitea/setup b/docker/root/etc/s6/gitea/setup
+index f87ce9115..565bfcba9 100755
+--- a/docker/root/etc/s6/gitea/setup
+++ b/docker/root/etc/s6/gitea/setup
+@@ -44,6 +44,8 @@ if [ ! -f ${GITEA_CUSTOM}/conf/app.ini ]; then
+     SECRET_KEY=${SECRET_KEY:-""} \
+     envsubst < /etc/templates/app.ini > ${GITEA_CUSTOM}/conf/app.ini
+
+    /app/gitea/environment-to-ini -c ${GITEA_CUSTOM}/conf/app.ini
+
+     chown ${USER}:git ${GITEA_CUSTOM}/conf/app.ini
+ fi
+
--- a/contrib/environment-to-ini/environment-to-ini.go
+++ b/contrib/environment-to-ini/environment-to-ini.go
@ -0,0 +1,224 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/unknwon/com"
+	"github.com/urfave/cli"
+	ini "gopkg.in/ini.v1"
+)
+
+// EnvironmentPrefix environment variables prefixed with this represent ini values to write
+const EnvironmentPrefix = "GITEA"
+
+func main() {
+	app := cli.NewApp()
+	app.Name = "environment-to-ini"
+	app.Usage = "Use provided environment to update configuration ini"
+	app.Description = `As a helper to allow docker users to update the gitea configuration
+	through the environment, this command allows environment variables to
+	be mapped to values in the ini.
+
+	Environment variables of the form "GITEA__SECTION_NAME__KEY_NAME"
+	will be mapped to the ini section "[section_name]" and the key
+	"KEY_NAME" with the value as provided.
+
+	Environment variables are usually restricted to a reduced character
+	set "0-9A-Z_" - in order to allow the setting of sections with
+	characters outside of that set, they should be escaped as following:
+	"_0X2E_" for ".". The entire section and key names can be escaped as
+	a UTF8 byte string if necessary. E.g. to configure:
+
+		"""
+		...
+		[log.console]
+		COLORIZE=false
+		STDERR=true
+		...
+		"""
+
+	You would set the environment variables: "GITEA__LOG_0x2E_CONSOLE__COLORIZE=false"
+	and "GITEA__LOG_0x2E_CONSOLE__STDERR=false". Other examples can be found
+	on the configuration cheat sheet.`
+	app.Flags = []cli.Flag{
+		cli.StringFlag{
+			Name:  "custom-path, C",
+			Value: setting.CustomPath,
+			Usage: "Custom path file path",
+		},
+		cli.StringFlag{
+			Name:  "config, c",
+			Value: setting.CustomConf,
+			Usage: "Custom configuration file path",
+		},
+		cli.StringFlag{
+			Name:  "work-path, w",
+			Value: setting.AppWorkPath,
+			Usage: "Set the gitea working path",
+		},
+		cli.StringFlag{
+			Name:  "out, o",
+			Value: "",
+			Usage: "Destination file to write to",
+		},
+		cli.BoolFlag{
+			Name:  "clear",
+			Usage: "Clears the matched variables from the environment",
+		},
+		cli.StringFlag{
+			Name:  "prefix, p",
+			Value: EnvironmentPrefix,
+			Usage: "Environment prefix to look for - will be suffixed by __ (2 underscores)",
+		},
+	}
+	app.Action = runEnvironmentToIni
+	setting.SetCustomPathAndConf("", "", "")
+
+	err := app.Run(os.Args)
+	if err != nil {
+		log.Fatal("Failed to run app with %s: %v", os.Args, err)
+	}
+}
+
+func runEnvironmentToIni(c *cli.Context) error {
+	providedCustom := c.String("custom-path")
+	providedConf := c.String("config")
+	providedWorkPath := c.String("work-path")
+	setting.SetCustomPathAndConf(providedCustom, providedConf, providedWorkPath)
+
+	cfg := ini.Empty()
+	if com.IsFile(setting.CustomConf) {
+		if err := cfg.Append(setting.CustomConf); err != nil {
+			log.Fatal("Failed to load custom conf '%s': %v", setting.CustomConf, err)
+		}
+	} else {
+		log.Warn("Custom config '%s' not found, ignore this if you're running first time", setting.CustomConf)
+	}
+	cfg.NameMapper = ini.AllCapsUnderscore
+
+	prefix := c.String("prefix") + "__"
+
+	for _, kv := range os.Environ() {
+		idx := strings.IndexByte(kv, '=')
+		if idx < 0 {
+			continue
+		}
+		eKey := kv[:idx]
+		value := kv[idx+1:]
+		if !strings.HasPrefix(eKey, prefix) {
+			continue
+		}
+		eKey = eKey[len(prefix):]
+		sectionName, keyName := DecodeSectionKey(eKey)
+		if len(keyName) == 0 {
+			continue
+		}
+		section, err := cfg.GetSection(sectionName)
+		if err != nil {
+			section, err = cfg.NewSection(sectionName)
+			if err != nil {
+				log.Error("Error creating section: %s : %v", sectionName, err)
+				continue
+			}
+		}
+		key := section.Key(keyName)
+		if key == nil {
+			key, err = section.NewKey(keyName, value)
+			if err != nil {
+				log.Error("Error creating key: %s in section: %s with value: %s : %v", keyName, sectionName, value, err)
+				continue
+			}
+		}
+		key.SetValue(value)
+	}
+	destination := c.String("out")
+	if len(destination) == 0 {
+		destination = setting.CustomConf
+	}
+	err := cfg.SaveTo(destination)
+	if err != nil {
+		return err
+	}
+	if c.Bool("clear") {
+		for _, kv := range os.Environ() {
+			idx := strings.IndexByte(kv, '=')
+			if idx < 0 {
+				continue
+			}
+			eKey := kv[:idx]
+			if strings.HasPrefix(eKey, prefix) {
+				_ = os.Unsetenv(eKey)
+			}
+		}
+	}
+	return nil
+}
+
+const escapeRegexpString = "_0[xX](([0-9a-fA-F][0-9a-fA-F])+)_"
+
+var escapeRegex = regexp.MustCompile(escapeRegexpString)
+
+// DecodeSectionKey will decode a portable string encoded Section__Key pair
+// Portable strings are considered to be of the form [A-Z0-9_]*
+// We will encode a disallowed value as the UTF8 byte string preceded by _0X and
+// followed by _. E.g. _0X2C_ for a '-' and _0X2E_ for '.'
+// Section and Key are separated by a plain '__'.
+// The entire section can be encoded as a UTF8 byte string
+func DecodeSectionKey(encoded string) (string, string) {
+	section := ""
+	key := ""
+
+	inKey := false
+	last := 0
+	escapeStringIndices := escapeRegex.FindAllStringIndex(encoded, -1)
+	for _, unescapeIdx := range escapeStringIndices {
+		preceding := encoded[last:unescapeIdx[0]]
+		if !inKey {
+			if splitter := strings.Index(preceding, "__"); splitter > -1 {
+				section += preceding[:splitter]
+				inKey = true
+				key += preceding[splitter+2:]
+			} else {
+				section += preceding
+			}
+		} else {
+			key += preceding
+		}
+		toDecode := encoded[unescapeIdx[0]+3 : unescapeIdx[1]-1]
+		decodedBytes := make([]byte, len(toDecode)/2)
+		for i := 0; i < len(toDecode)/2; i++ {
+			// Can ignore error here as we know these should be hexadecimal from the regexp
+			byteInt, _ := strconv.ParseInt(toDecode[2*i:2*i+2], 16, 0)
+			decodedBytes[i] = byte(byteInt)
+		}
+		if inKey {
+			key += string(decodedBytes)
+		} else {
+			section += string(decodedBytes)
+		}
+		last = unescapeIdx[1]
+	}
+	remaining := encoded[last:]
+	if !inKey {
+		if splitter := strings.Index(remaining, "__"); splitter > -1 {
+			section += remaining[:splitter]
+			inKey = true
+			key += remaining[splitter+2:]
+		} else {
+			section += remaining
+		}
+	} else {
+		key += remaining
+	}
+	return section, key
+}