1	/*
     2	Copyright 2011 The Perkeep Authors.
     3	
     4	Licensed under the Apache License, Version 2.0 (the "License");
     5	you may not use this file except in compliance with the License.
     6	You may obtain a copy of the License at
     7	
     8	     http://www.apache.org/licenses/LICENSE-2.0
     9	
    10	Unless required by applicable law or agreed to in writing, software
    11	distributed under the License is distributed on an "AS IS" BASIS,
    12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13	See the License for the specific language governing permissions and
    14	limitations under the License.
    15	*/
    16	
    17	package client
    18	
    19	import (
    20		"errors"
    21		"flag"
    22		"fmt"
    23		"log"
    24		"os"
    25		"path/filepath"
    26		"strconv"
    27		"strings"
    28		"sync"
    29	
    30		"go4.org/jsonconfig"
    31		"perkeep.org/internal/osutil"
    32		"perkeep.org/pkg/auth"
    33		"perkeep.org/pkg/blob"
    34		"perkeep.org/pkg/buildinfo"
    35		"perkeep.org/pkg/client/android"
    36		"perkeep.org/pkg/env"
    37		"perkeep.org/pkg/jsonsign"
    38		"perkeep.org/pkg/types/camtypes"
    39		"perkeep.org/pkg/types/clientconfig"
    40	
    41		"go4.org/wkfs"
    42	)
    43	
    44	// If set, flagServer overrides the JSON config file
    45	// ~/.config/perkeep/client-config.json
    46	// (i.e. osutil.UserClientConfigPath()) "server" key.
    47	//
    48	// A main binary must call AddFlags to expose it.
    49	var flagServer string
    50	
    51	// AddFlags registers the "server" and "secret-keyring" string flags.
    52	func AddFlags() {
    53		defaultPath := "/x/y/z/we're/in-a-test"
    54		if !buildinfo.TestingLinked() {
    55			defaultPath = osutil.UserClientConfigPath()
    56		}
    57		flag.StringVar(&flagServer, "server", "", "Perkeep server prefix. If blank, the default from the \"server\" field of "+defaultPath+" is used. Acceptable forms: https://you.example.com, example.com:1345 (https assumed), or http://you.example.com/alt-root")
    58		osutil.AddSecretRingFlag()
    59	}
    60	
    61	// ExplicitServer returns the Perkeep server given in the "server"
    62	// flag, if any.
    63	//
    64	// Use AddFlags to register the flag before any flag.Parse call.
    65	func ExplicitServer() string {
    66		return flagServer
    67	}
    68	
    69	var (
    70		configOnce sync.Once
    71		config     *clientconfig.Config
    72	
    73		configDisabled, _ = strconv.ParseBool(os.Getenv("CAMLI_DISABLE_CLIENT_CONFIG_FILE"))
    74	)
    75	
    76	// config parsing in the global environment.
    77	func parseConfig() {
    78		var nilClient *Client
    79		nilClient.parseConfig()
    80	}
    81	
    82	// lazy config parsing when there's a known client already.
    83	// The client c may be nil.
    84	func (c *Client) parseConfig() {
    85		if android.OnAndroid() {
    86			panic("parseConfig should never have been called on Android")
    87		}
    88		if configDisabled {
    89			panic("parseConfig should never have been called with CAMLI_DISABLE_CLIENT_CONFIG_FILE set")
    90		}
    91		configPath := osutil.UserClientConfigPath()
    92		if _, err := wkfs.Stat(configPath); os.IsNotExist(err) {
    93			if c != nil && c.isSharePrefix {
    94				return
    95			}
    96			errMsg := fmt.Sprintf("Client configuration file %v does not exist. See 'pk-put init' to generate it.", configPath)
    97			if keyID := serverKeyId(); keyID != "" {
    98				hint := fmt.Sprintf("\nThe key id %v was found in the server config %v, so you might want:\n'pk-put init -gpgkey %v'", keyID, osutil.UserServerConfigPath(), keyID)
    99				errMsg += hint
   100			}
   101			log.Fatal(errMsg)
   102		}
   103		// TODO: instead of using jsonconfig, we could read the file,
   104		// and unmarshal into the structs that we now have in
   105		// pkg/types/clientconfig. But we'll have to add the old
   106		// fields (before the name changes, and before the
   107		// multi-servers change) to the structs as well for our
   108		// graceful conversion/error messages to work.
   109		conf, err := osutil.NewJSONConfigParser().ReadFile(configPath)
   110		if err != nil {
   111			log.Fatal(err.Error())
   112		}
   113		cfg := jsonconfig.Obj(conf)
   114	
   115		if singleServerAuth := cfg.OptionalString("auth", ""); singleServerAuth != "" {
   116			newConf, err := convertToMultiServers(cfg)
   117			if err != nil {
   118				log.Print(err)
   119			} else {
   120				cfg = newConf
   121			}
   122		}
   123	
   124		config = &clientconfig.Config{
   125			Identity:           cfg.OptionalString("identity", ""),
   126			IdentitySecretRing: cfg.OptionalString("identitySecretRing", ""),
   127			IgnoredFiles:       cfg.OptionalList("ignoredFiles"),
   128		}
   129		serversList := make(map[string]*clientconfig.Server)
   130		servers := cfg.OptionalObject("servers")
   131		for alias, vei := range servers {
   132			// An alias should never be confused with a host name,
   133			// so we forbid anything looking like one.
   134			if isURLOrHostPort(alias) {
   135				log.Fatalf("Server alias %q looks like a hostname; \".\" or \";\" are not allowed.", alias)
   136			}
   137			serverMap, ok := vei.(map[string]interface{})
   138			if !ok {
   139				log.Fatalf("entry %q in servers section is a %T, want an object", alias, vei)
   140			}
   141			serverConf := jsonconfig.Obj(serverMap)
   142			serverStr, err := cleanServer(serverConf.OptionalString("server", ""))
   143			if err != nil {
   144				log.Fatalf("invalid server alias %q: %v", alias, err)
   145			}
   146			server := &clientconfig.Server{
   147				Server:       serverStr,
   148				Auth:         serverConf.OptionalString("auth", ""),
   149				IsDefault:    serverConf.OptionalBool("default", false),
   150				TrustedCerts: serverConf.OptionalList("trustedCerts"),
   151			}
   152			if err := serverConf.Validate(); err != nil {
   153				log.Fatalf("Error in servers section of config file for server %q: %v", alias, err)
   154			}
   155			serversList[alias] = server
   156		}
   157		config.Servers = serversList
   158		if err := cfg.Validate(); err != nil {
   159			printConfigChangeHelp(cfg)
   160			log.Fatalf("Error in config file: %v", err)
   161		}
   162	}
   163	
   164	// isURLOrHostPort returns true if s looks like a URL, or a hostname, i.e it starts with a scheme and/or it contains a period or a colon.
   165	func isURLOrHostPort(s string) bool {
   166		return strings.HasPrefix(s, "http://") ||
   167			strings.HasPrefix(s, "https://") ||
   168			strings.Contains(s, ".") || strings.Contains(s, ":")
   169	}
   170	
   171	// convertToMultiServers takes an old style single-server client configuration and maps it to new a multi-servers configuration that is returned.
   172	func convertToMultiServers(conf jsonconfig.Obj) (jsonconfig.Obj, error) {
   173		server := conf.OptionalString("server", "")
   174		if server == "" {
   175			return nil, errors.New("could not convert config to multi-servers style: no \"server\" key found")
   176		}
   177		newConf := jsonconfig.Obj{
   178			"servers": map[string]interface{}{
   179				"server": map[string]interface{}{
   180					"auth":    conf.OptionalString("auth", ""),
   181					"default": true,
   182					"server":  server,
   183				},
   184			},
   185			"identity":           conf.OptionalString("identity", ""),
   186			"identitySecretRing": conf.OptionalString("identitySecretRing", ""),
   187		}
   188		if ignoredFiles := conf.OptionalList("ignoredFiles"); ignoredFiles != nil {
   189			var list []interface{}
   190			for _, v := range ignoredFiles {
   191				list = append(list, v)
   192			}
   193			newConf["ignoredFiles"] = list
   194		}
   195		return newConf, nil
   196	}
   197	
   198	// printConfigChangeHelp checks if conf contains obsolete keys,
   199	// and prints additional help in this case.
   200	func printConfigChangeHelp(conf jsonconfig.Obj) {
   201		// rename maps from old key names to the new ones.
   202		// If there is no new one, the value is the empty string.
   203		rename := map[string]string{
   204			"keyId":            "identity",
   205			"publicKeyBlobref": "",
   206			"selfPubKeyDir":    "",
   207			"secretRing":       "identitySecretRing",
   208		}
   209		oldConfig := false
   210		configChangedMsg := fmt.Sprintf("The client configuration file (%s) keys have changed.\n", osutil.UserClientConfigPath())
   211		for _, unknown := range conf.UnknownKeys() {
   212			v, ok := rename[unknown]
   213			if ok {
   214				if v != "" {
   215					configChangedMsg += fmt.Sprintf("%q should be renamed %q.\n", unknown, v)
   216				} else {
   217					configChangedMsg += fmt.Sprintf("%q should be removed.\n", unknown)
   218				}
   219				oldConfig = true
   220			}
   221		}
   222		if oldConfig {
   223			configChangedMsg += "Please see https://perkeep.org/doc/client-config, or use pk-put init to recreate a default one."
   224			log.Print(configChangedMsg)
   225		}
   226	}
   227	
   228	// serverKeyId returns the public gpg key id ("identity" field)
   229	// from the user's server config , if any.
   230	// It returns the empty string otherwise.
   231	func serverKeyId() string {
   232		serverConfigFile := osutil.UserServerConfigPath()
   233		if _, err := wkfs.Stat(serverConfigFile); err != nil {
   234			if os.IsNotExist(err) {
   235				return ""
   236			}
   237			log.Fatalf("Could not stat %v: %v", serverConfigFile, err)
   238		}
   239		obj, err := jsonconfig.ReadFile(serverConfigFile)
   240		if err != nil {
   241			return ""
   242		}
   243		keyID, ok := obj["identity"].(string)
   244		if !ok {
   245			return ""
   246		}
   247		return keyID
   248	}
   249	
   250	// cleanServer returns the canonical URL of the provided server, which must be a URL, IP, host (with dot), or host/ip:port.
   251	// The returned canonical URL will have trailing slashes removed and be prepended with "https://" if no scheme is provided.
   252	func cleanServer(server string) (string, error) {
   253		if !isURLOrHostPort(server) {
   254			return "", fmt.Errorf("server %q does not look like a server address and could be confused with a server alias. It should look like [http[s]://]foo[.com][:port] with at least one of the optional parts.", server)
   255		}
   256		// Remove trailing slash if provided.
   257		server = strings.TrimSuffix(server, "/")
   258	
   259		// Default to "https://" when not specified
   260		if !strings.HasPrefix(server, "http://") && !strings.HasPrefix(server, "https://") {
   261			server = "https://" + server
   262		}
   263		return server, nil
   264	}
   265	
   266	// getServer returns the server's URL found either as a command-line flag,
   267	// or as the default server in the config file.
   268	func getServer() (string, error) {
   269		if s := os.Getenv("CAMLI_SERVER"); s != "" {
   270			return cleanServer(s)
   271		}
   272		if flagServer != "" {
   273			if !isURLOrHostPort(flagServer) {
   274				configOnce.Do(parseConfig)
   275				serverConf, ok := config.Servers[flagServer]
   276				if ok {
   277					return serverConf.Server, nil
   278				}
   279				log.Printf("%q looks like a server alias, but no such alias found in config.", flagServer)
   280			} else {
   281				return cleanServer(flagServer)
   282			}
   283		}
   284		server, err := defaultServer()
   285		if err != nil {
   286			return "", err
   287		}
   288		if server == "" {
   289			return "", camtypes.ErrClientNoServer
   290		}
   291		return cleanServer(server)
   292	}
   293	
   294	func defaultServer() (string, error) {
   295		configOnce.Do(parseConfig)
   296		wantAlias := os.Getenv("CAMLI_DEFAULT_SERVER")
   297		for alias, serverConf := range config.Servers {
   298			if (wantAlias != "" && wantAlias == alias) || (wantAlias == "" && serverConf.IsDefault) {
   299				return cleanServer(serverConf.Server)
   300			}
   301		}
   302		return "", nil
   303	}
   304	
   305	func (c *Client) useTLS() bool {
   306		return strings.HasPrefix(c.discoRoot(), "https://")
   307	}
   308	
   309	// SetupAuth sets the client's authMode. It tries from the environment first if we're on android or in dev mode, and then from the client configuration.
   310	func (c *Client) SetupAuth() error {
   311		if c.noExtConfig {
   312			if c.authMode != nil {
   313				if _, ok := c.authMode.(*auth.None); !ok {
   314					return nil
   315				}
   316			}
   317			return errors.New("client: noExtConfig set; auth should not be configured from config or env vars")
   318		}
   319		// env var takes precedence, but only if we're in dev mode or on android.
   320		// Too risky otherwise.
   321		if android.OnAndroid() ||
   322			env.IsDev() ||
   323			configDisabled {
   324			authMode, err := auth.FromEnv()
   325			if err == nil {
   326				c.authMode = authMode
   327				return nil
   328			}
   329			if !errors.Is(err, auth.ErrNoAuth) {
   330				return fmt.Errorf("Could not set up auth from env var CAMLI_AUTH: %w", err)
   331			}
   332		}
   333		if c.server == "" {
   334			return fmt.Errorf("no server defined for this client: can not set up auth")
   335		}
   336		authConf := serverAuth(c.server)
   337		if authConf == "" {
   338			c.authErr = fmt.Errorf("could not find auth key for server %q in config, defaulting to no auth", c.server)
   339			c.authMode = auth.None{}
   340			return nil
   341		}
   342		var err error
   343		c.authMode, err = auth.FromConfig(authConf)
   344		return err
   345	}
   346	
   347	// serverAuth returns the auth scheme for server from the config, or the empty string if the server was not found in the config.
   348	func serverAuth(server string) string {
   349		configOnce.Do(parseConfig)
   350		alias := config.Alias(server)
   351		if alias == "" {
   352			return ""
   353		}
   354		return config.Servers[alias].Auth
   355	}
   356	
   357	// SetupAuthFromString configures the clients authentication mode from
   358	// an explicit auth string.
   359	func (c *Client) SetupAuthFromString(a string) error {
   360		// TODO(mpl): review the one using that (pkg/blobserver/remote/remote.go)
   361		var err error
   362		c.authMode, err = auth.FromConfig(a)
   363		return err
   364	}
   365	
   366	// SecretRingFile returns the filename to the user's GPG secret ring.
   367	// The value comes from either the --secret-keyring flag, the
   368	// CAMLI_SECRET_RING environment variable, the client config file's
   369	// "identitySecretRing" value, or the operating system default location.
   370	func (c *Client) SecretRingFile() string {
   371		if osutil.HasSecretRingFlag() {
   372			if secretRing, ok := osutil.ExplicitSecretRingFile(); ok {
   373				return secretRing
   374			}
   375		}
   376		if android.OnAndroid() {
   377			panic("on android, so CAMLI_SECRET_RING should have been defined, or --secret-keyring used.")
   378		}
   379		if c.noExtConfig {
   380			log.Print("client: noExtConfig set; cannot get secret ring file from config or env vars.")
   381			return ""
   382		}
   383		if configDisabled {
   384			panic("Need a secret ring, and config file disabled")
   385		}
   386		configOnce.Do(parseConfig)
   387		if config.IdentitySecretRing == "" {
   388			return osutil.SecretRingFile()
   389		}
   390		return config.IdentitySecretRing
   391	}
   392	
   393	func fileExists(name string) bool {
   394		_, err := os.Stat(name)
   395		return err == nil
   396	}
   397	
   398	// SignerPublicKeyBlobref returns the blobref of signer's public key.
   399	// The blobref may not be valid (zero blob.Ref) if e.g the configuration
   400	// is invalid or incomplete.
   401	func (c *Client) SignerPublicKeyBlobref() blob.Ref {
   402		c.initSignerPublicKeyBlobrefOnce.Do(c.initSignerPublicKeyBlobref)
   403		return c.signerPublicKeyRef
   404	}
   405	
   406	func (c *Client) initSignerPublicKeyBlobref() {
   407		if c.noExtConfig {
   408			log.Print("client: noExtConfig set; cannot get public key from config or env vars.")
   409			return
   410		}
   411		keyID := os.Getenv("CAMLI_KEYID")
   412		if keyID == "" {
   413			configOnce.Do(parseConfig)
   414			keyID = config.Identity
   415			if keyID == "" {
   416				log.Fatalf("No 'identity' key in JSON configuration file %q; have you run \"pk-put init\"?", osutil.UserClientConfigPath())
   417			}
   418		}
   419		keyRing := c.SecretRingFile()
   420		if !fileExists(keyRing) {
   421			log.Fatalf("Could not find keyID %q, because secret ring file %q does not exist.", keyID, keyRing)
   422		}
   423		entity, err := jsonsign.EntityFromSecring(keyID, keyRing)
   424		if err != nil {
   425			log.Fatalf("Couldn't find keyID %q in secret ring %v: %v", keyID, keyRing, err)
   426		}
   427		armored, err := jsonsign.ArmoredPublicKey(entity)
   428		if err != nil {
   429			log.Fatalf("Error serializing public key: %v", err)
   430		}
   431	
   432		c.signerPublicKeyRef = blob.RefFromString(armored)
   433		c.publicKeyArmored = armored
   434	}
   435	
   436	func (c *Client) initTrustedCerts() {
   437		if c.noExtConfig {
   438			return
   439		}
   440		if e := os.Getenv("CAMLI_TRUSTED_CERT"); e != "" {
   441			c.trustedCerts = strings.Split(e, ",")
   442			return
   443		}
   444		c.trustedCerts = []string{}
   445		if android.OnAndroid() || configDisabled {
   446			return
   447		}
   448		if c.server == "" {
   449			log.Printf("No server defined: can not define trustedCerts for this client.")
   450			return
   451		}
   452		trustedCerts := c.serverTrustedCerts(c.server)
   453		if trustedCerts == nil {
   454			return
   455		}
   456		for _, trustedCert := range trustedCerts {
   457			c.trustedCerts = append(c.trustedCerts, strings.ToLower(trustedCert))
   458		}
   459	}
   460	
   461	// serverTrustedCerts returns the trusted certs for server from the config.
   462	func (c *Client) serverTrustedCerts(server string) []string {
   463		configOnce.Do(c.parseConfig)
   464		if config == nil {
   465			return nil
   466		}
   467		alias := config.Alias(server)
   468		if alias == "" {
   469			return nil
   470		}
   471		return config.Servers[alias].TrustedCerts
   472	}
   473	
   474	func (c *Client) getTrustedCerts() []string {
   475		c.initTrustedCertsOnce.Do(c.initTrustedCerts)
   476		return c.trustedCerts
   477	}
   478	
   479	func (c *Client) initIgnoredFiles() {
   480		defer func() {
   481			c.ignoreChecker = newIgnoreChecker(c.ignoredFiles)
   482		}()
   483		if c.noExtConfig {
   484			return
   485		}
   486		if e := os.Getenv("CAMLI_IGNORED_FILES"); e != "" {
   487			c.ignoredFiles = strings.Split(e, ",")
   488			return
   489		}
   490		c.ignoredFiles = []string{}
   491		if android.OnAndroid() || configDisabled {
   492			return
   493		}
   494		configOnce.Do(parseConfig)
   495		c.ignoredFiles = config.IgnoredFiles
   496	}
   497	
   498	var osutilHomeDir = osutil.HomeDir // changed by tests
   499	
   500	// newIgnoreChecker uses ignoredFiles to build and return a func that returns whether the file path argument should be ignored. See IsIgnoredFile for the ignore rules.
   501	func newIgnoreChecker(ignoredFiles []string) func(path string) (shouldIgnore bool) {
   502		var fns []func(string) bool
   503	
   504		// copy of ignoredFiles for us to mutate
   505		ignFiles := append([]string(nil), ignoredFiles...)
   506		for k, v := range ignFiles {
   507			if strings.HasPrefix(v, filepath.FromSlash("~/")) {
   508				ignFiles[k] = filepath.Join(osutilHomeDir(), v[2:])
   509			}
   510		}
   511		// We cache the ignoredFiles patterns in 3 categories (not necessarily exclusive):
   512		// 1) shell patterns
   513		// 3) absolute paths
   514		// 4) paths components
   515		for _, pattern := range ignFiles {
   516			pattern := pattern
   517			_, err := filepath.Match(pattern, "whatever")
   518			if err == nil {
   519				fns = append(fns, func(v string) bool { return isShellPatternMatch(pattern, v) })
   520			}
   521		}
   522		for _, pattern := range ignFiles {
   523			pattern := pattern
   524			if filepath.IsAbs(pattern) {
   525				fns = append(fns, func(v string) bool { return hasDirPrefix(filepath.Clean(pattern), v) })
   526			} else {
   527				fns = append(fns, func(v string) bool { return hasComponent(filepath.Clean(pattern), v) })
   528			}
   529		}
   530	
   531		return func(path string) bool {
   532			for _, fn := range fns {
   533				if fn(path) {
   534					return true
   535				}
   536			}
   537			return false
   538		}
   539	}
   540	
   541	var filepathSeparatorString = string(filepath.Separator)
   542	
   543	// isShellPatternMatch returns whether fullpath matches the shell pattern, as defined by http://golang.org/pkg/path/filepath/#Match. As an additional special case, when the pattern looks like a basename, the last path element of fullpath is also checked against it.
   544	func isShellPatternMatch(shellPattern, fullpath string) bool {
   545		match, _ := filepath.Match(shellPattern, fullpath)
   546		if match {
   547			return true
   548		}
   549		if !strings.Contains(shellPattern, filepathSeparatorString) {
   550			match, _ := filepath.Match(shellPattern, filepath.Base(fullpath))
   551			if match {
   552				return true
   553			}
   554		}
   555		return false
   556	}
   557	
   558	// hasDirPrefix reports whether the path has the provided directory prefix.
   559	// Both should be absolute paths.
   560	func hasDirPrefix(dirPrefix, fullpath string) bool {
   561		if !strings.HasPrefix(fullpath, dirPrefix) {
   562			return false
   563		}
   564		if len(fullpath) == len(dirPrefix) {
   565			return true
   566		}
   567		if fullpath[len(dirPrefix)] == filepath.Separator {
   568			return true
   569		}
   570		return false
   571	}
   572	
   573	// hasComponent returns whether the pathComponent is a path component of
   574	// fullpath. i.e it is a part of fullpath that fits exactly between two path
   575	// separators.
   576	func hasComponent(component, fullpath string) bool {
   577		// trim Windows volume name
   578		fullpath = strings.TrimPrefix(fullpath, filepath.VolumeName(fullpath))
   579		for {
   580			i := strings.Index(fullpath, component)
   581			if i == -1 {
   582				return false
   583			}
   584			if i != 0 && fullpath[i-1] == filepath.Separator {
   585				componentEnd := i + len(component)
   586				if componentEnd == len(fullpath) {
   587					return true
   588				}
   589				if fullpath[componentEnd] == filepath.Separator {
   590					return true
   591				}
   592			}
   593			fullpath = fullpath[i+1:]
   594		}
   595	}