1	/*
     2	Copyright 2011 The Perkeep Authors
     3	
     4	Licensed under the Apache License, Version 2.0 (the "License");
     5	you may not use this file except in compliance with the License.
     6	You may obtain a copy of the License at
     7	
     8	     http://www.apache.org/licenses/LICENSE-2.0
     9	
    10	Unless required by applicable law or agreed to in writing, software
    11	distributed under the License is distributed on an "AS IS" BASIS,
    12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13	See the License for the specific language governing permissions and
    14	limitations under the License.
    15	*/
    16	
    17	// The perkeepd binary is the Perkeep server.
    18	package main // import "perkeep.org/server/perkeepd"
    19	
    20	import (
    21		"context"
    22		"flag"
    23		"fmt"
    24		"io"
    25		"log"
    26		"net"
    27		"net/http"
    28		"os"
    29		"os/signal"
    30		"path/filepath"
    31		"runtime"
    32		"strconv"
    33		"strings"
    34		"syscall"
    35		"time"
    36	
    37		"perkeep.org/internal/geocode"
    38		"perkeep.org/internal/httputil"
    39		"perkeep.org/internal/netutil"
    40		"perkeep.org/internal/osutil"
    41		"perkeep.org/internal/osutil/gce"
    42		"perkeep.org/pkg/buildinfo"
    43		"perkeep.org/pkg/env"
    44		"perkeep.org/pkg/serverinit"
    45		"perkeep.org/pkg/webserver"
    46	
    47		// VM environments:
    48		// for init side-effects + LogWriter
    49	
    50		// Storage options:
    51		_ "perkeep.org/pkg/blobserver/azure"
    52		"perkeep.org/pkg/blobserver/blobpacked"
    53		_ "perkeep.org/pkg/blobserver/cond"
    54		_ "perkeep.org/pkg/blobserver/diskpacked"
    55		_ "perkeep.org/pkg/blobserver/encrypt"
    56		_ "perkeep.org/pkg/blobserver/google/cloudstorage"
    57		_ "perkeep.org/pkg/blobserver/google/drive"
    58		_ "perkeep.org/pkg/blobserver/localdisk"
    59		_ "perkeep.org/pkg/blobserver/mongo"
    60		_ "perkeep.org/pkg/blobserver/overlay"
    61		_ "perkeep.org/pkg/blobserver/proxycache"
    62		_ "perkeep.org/pkg/blobserver/remote"
    63		_ "perkeep.org/pkg/blobserver/replica"
    64		_ "perkeep.org/pkg/blobserver/s3"
    65		_ "perkeep.org/pkg/blobserver/shard"
    66		_ "perkeep.org/pkg/blobserver/union"
    67	
    68		// Indexers: (also present themselves as storage targets)
    69		// KeyValue implementations:
    70		_ "perkeep.org/pkg/sorted/kvfile"
    71		_ "perkeep.org/pkg/sorted/leveldb"
    72		_ "perkeep.org/pkg/sorted/mongo"
    73		_ "perkeep.org/pkg/sorted/mysql"
    74		_ "perkeep.org/pkg/sorted/postgres"
    75	
    76		// Handlers:
    77		_ "perkeep.org/pkg/search"
    78		_ "perkeep.org/pkg/server" // UI, publish, etc
    79	
    80		// Importers:
    81		_ "perkeep.org/pkg/importer/allimporters"
    82	
    83		// Licence:
    84		_ "perkeep.org/pkg/camlegal"
    85	
    86		"go4.org/legal"
    87		"go4.org/wkfs"
    88	
    89		"golang.org/x/crypto/acme/autocert"
    90	)
    91	
    92	var (
    93		flagVersion    = flag.Bool("version", false, "show version")
    94		flagHelp       = flag.Bool("help", false, "show usage")
    95		flagLegal      = flag.Bool("legal", false, "show licenses")
    96		flagConfigFile = flag.String("configfile", "",
    97			"Config file to use, relative to the Perkeep configuration directory root. "+
    98				"If blank, the default is used or auto-generated. "+
    99				"If it starts with 'http:' or 'https:', it is fetched from the network.")
   100		flagListen      = flag.String("listen", "", "host:port to listen on, or :0 to auto-select. If blank, the value in the config will be used instead.")
   101		flagOpenBrowser = flag.Bool("openbrowser", true, "Launches the UI on startup")
   102		flagReindex     = flag.Bool("reindex", false, "Reindex all blobs on startup")
   103		flagRecovery    = flag.Int("recovery", 0, "Recovery mode: it corresponds for now to the recovery modes of the blobpacked package. Which means: 0 does nothing, 1 rebuilds the blobpacked index without erasing it, and 2 wipes the blobpacked index before rebuilding it.")
   104		flagSyslog      = flag.Bool("syslog", false, "Log everything only to syslog. It is an error to use this flag on windows.")
   105		flagKeepGoing   = flag.Bool("keep-going", false, "Continue after reindex or blobpacked recovery errors")
   106		flagPollParent  bool
   107	)
   108	
   109	func init() {
   110		if debug, _ := strconv.ParseBool(os.Getenv("CAMLI_MORE_FLAGS")); debug {
   111			flag.BoolVar(&flagPollParent, "pollparent", false, "Perkeepd regularly polls its parent process to detect if it has been orphaned, and terminates in that case. Mainly useful for tests.")
   112		}
   113	}
   114	
   115	func exitf(pattern string, args ...interface{}) {
   116		if !strings.HasSuffix(pattern, "\n") {
   117			pattern = pattern + "\n"
   118		}
   119		fmt.Fprintf(os.Stderr, pattern, args...)
   120		os.Exit(1)
   121	}
   122	
   123	func slurpURL(urls string, limit int64) ([]byte, error) {
   124		res, err := http.Get(urls)
   125		if err != nil {
   126			return nil, err
   127		}
   128		defer res.Body.Close()
   129		return io.ReadAll(io.LimitReader(res.Body, limit))
   130	}
   131	
   132	// loadConfig returns the server's parsed config file, locating it using the provided arg.
   133	//
   134	// The arg may be of the form:
   135	//   - empty, to mean automatic (will write a default high-level config if
   136	//     no cloud config is available)
   137	//   - a filepath absolute or relative to the user's configuration directory,
   138	//   - a URL
   139	func loadConfig(arg string) (*serverinit.Config, error) {
   140		if strings.HasPrefix(arg, "http://") || strings.HasPrefix(arg, "https://") {
   141			contents, err := slurpURL(arg, 256<<10)
   142			if err != nil {
   143				return nil, err
   144			}
   145			return serverinit.Load(contents)
   146		}
   147		var absPath string
   148		switch {
   149		case arg == "":
   150			absPath = osutil.UserServerConfigPath()
   151			_, err := wkfs.Stat(absPath)
   152			if err == nil {
   153				break
   154			}
   155			if !os.IsNotExist(err) {
   156				return nil, err
   157			}
   158			conf, err := serverinit.DefaultEnvConfig()
   159			if err != nil || conf != nil {
   160				return conf, err
   161			}
   162			configDir, err := osutil.PerkeepConfigDir()
   163			if err != nil {
   164				return nil, err
   165			}
   166			if err := wkfs.MkdirAll(configDir, 0700); err != nil {
   167				return nil, err
   168			}
   169			log.Printf("Generating template config file %s", absPath)
   170			if err := serverinit.WriteDefaultConfigFile(absPath); err != nil {
   171				return nil, err
   172			}
   173		case filepath.IsAbs(arg):
   174			absPath = arg
   175		default:
   176			configDir, err := osutil.PerkeepConfigDir()
   177			if err != nil {
   178				return nil, err
   179			}
   180			absPath = filepath.Join(configDir, arg)
   181		}
   182		return serverinit.LoadFile(absPath)
   183	}
   184	
   185	// If cert/key files are specified, and found, use them.
   186	// If cert/key files are specified, not found, and the default values, generate
   187	// them (self-signed CA used as a cert), and use them.
   188	// If cert/key files are not specified, use Let's Encrypt.
   189	func setupTLS(ws *webserver.Server, config *serverinit.Config, hostname string) {
   190		cert, key := config.HTTPSCert(), config.HTTPSKey()
   191		if !config.HTTPS() {
   192			return
   193		}
   194		if (cert != "") != (key != "") {
   195			exitf("httpsCert and httpsKey must both be either present or absent")
   196		}
   197		if config.IsTailscaleListener() {
   198			if cert != "" {
   199				exitf("httpsCert and httpsKey must be empty when using the tailscale listener")
   200			}
   201			// Call SetTLS to set enableTLS to true for now. The rest (the
   202			// GetCertificate hook) will be initialized later when tsnet is
   203			// initialized.
   204			ws.SetTLS(webserver.TLSSetup{})
   205			return
   206		}
   207	
   208		defCert := osutil.DefaultTLSCert()
   209		defKey := osutil.DefaultTLSKey()
   210		const hint = "You must add this certificate's fingerprint to your client's trusted certs list to use it. Like so:\n\"trustedCerts\": [\"%s\"],"
   211		if cert == defCert && key == defKey {
   212			_, err1 := wkfs.Stat(cert)
   213			_, err2 := wkfs.Stat(key)
   214			if err1 != nil || err2 != nil {
   215				if os.IsNotExist(err1) || os.IsNotExist(err2) {
   216					sig, err := httputil.GenSelfTLSFiles(hostname, defCert, defKey)
   217					if err != nil {
   218						exitf("Could not generate self-signed TLS cert: %q", err)
   219					}
   220					log.Printf(hint, sig)
   221				} else {
   222					exitf("Could not stat cert or key: %q, %q", err1, err2)
   223				}
   224			}
   225		}
   226		if cert == "" && key == "" {
   227			// Use Let's Encrypt if no files are specified, and we have a usable hostname.
   228			if netutil.IsFQDN(hostname) {
   229				m := autocert.Manager{
   230					Prompt:     autocert.AcceptTOS,
   231					HostPolicy: autocert.HostWhitelist(hostname),
   232					Cache:      autocert.DirCache(osutil.DefaultLetsEncryptCache()),
   233				}
   234				ws.SetTLS(webserver.TLSSetup{
   235					CertManager: m.GetCertificate,
   236				})
   237				log.Printf("Using Let's Encrypt tls-alpn-01 for %v", hostname)
   238				return
   239			}
   240			// Otherwise generate new certificates
   241			sig, err := httputil.GenSelfTLSFiles(hostname, defCert, defKey)
   242			if err != nil {
   243				exitf("Could not generate self signed creds: %q", err)
   244			}
   245			log.Printf(hint, sig)
   246			cert = defCert
   247			key = defKey
   248		}
   249		data, err := wkfs.ReadFile(cert)
   250		if err != nil {
   251			exitf("Failed to read pem certificate: %s", err)
   252		}
   253		sig, err := httputil.CertFingerprint(data)
   254		if err != nil {
   255			exitf("certificate error: %v", err)
   256		}
   257		log.Printf("TLS enabled, with SHA-256 certificate fingerprint: %v", sig)
   258		ws.SetTLS(webserver.TLSSetup{
   259			CertFile: cert,
   260			KeyFile:  key,
   261		})
   262	}
   263	
   264	func handleSignals(shutdownc <-chan io.Closer) {
   265		c := make(chan os.Signal, 1)
   266		signal.Notify(c, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM)
   267		for {
   268			sig := <-c
   269			sysSig, ok := sig.(syscall.Signal)
   270			if !ok {
   271				log.Fatal("Not a unix signal")
   272			}
   273			switch sysSig {
   274			case syscall.SIGHUP:
   275				log.Printf(`Got "%v" signal: restarting camli`, sig)
   276				err := osutil.RestartProcess()
   277				if err != nil {
   278					log.Fatal("Failed to restart: " + err.Error())
   279				}
   280			case syscall.SIGINT, syscall.SIGTERM:
   281				log.Printf(`Got "%v" signal: shutting down`, sig)
   282				donec := make(chan bool)
   283				go func() {
   284					cl := <-shutdownc
   285					if err := cl.Close(); err != nil {
   286						exitf("Error shutting down: %v", err)
   287					}
   288					donec <- true
   289				}()
   290				select {
   291				case <-donec:
   292					log.Printf("Shut down.")
   293					os.Exit(0)
   294				case <-time.After(2 * time.Second):
   295					exitf("Timeout shutting down. Exiting uncleanly.")
   296				}
   297			default:
   298				log.Fatal("Received another signal, should not happen.")
   299			}
   300		}
   301	}
   302	
   303	// listen discovers the listen address, base URL, and hostname that the ws is
   304	// going to use, sets up the TLS configuration, and starts listening.
   305	func listen(ws *webserver.Server, config *serverinit.Config) (baseURL string, err error) {
   306		baseURL = config.BaseURL()
   307	
   308		// Prefer the --listen flag value. Otherwise use the config value.
   309		listen := *flagListen
   310		if listen == "" {
   311			listen = config.ListenAddr()
   312		}
   313		if listen == "" {
   314			exitf("\"listen\" needs to be specified either in the config or on the command line")
   315		}
   316	
   317		if config.IsTailscaleListener() {
   318			setupTLS(ws, config, "unused-hostname")
   319		} else {
   320			hostname, err := certHostname(listen, baseURL)
   321			if err != nil {
   322				return "", fmt.Errorf("Bad baseURL or listen address: %w", err)
   323			}
   324			setupTLS(ws, config, hostname)
   325		}
   326	
   327		err = ws.Listen(listen)
   328		if err != nil {
   329			return "", fmt.Errorf("Listen: %w", err)
   330		}
   331		if baseURL == "" {
   332			baseURL = ws.ListenURL()
   333		}
   334		return baseURL, nil
   335	}
   336	
   337	// certHostname figures out the name to use for the TLS certificates, using baseURL
   338	// and falling back to the listen address if baseURL is empty or invalid.
   339	func certHostname(listen, baseURL string) (string, error) {
   340		hostPort, err := netutil.HostPort(baseURL)
   341		if err != nil {
   342			hostPort = listen
   343		}
   344		hostname, _, err := net.SplitHostPort(hostPort)
   345		if err != nil {
   346			return "", fmt.Errorf("failed to find hostname for cert from address %q (baseURL %q): %w", hostPort, baseURL, err)
   347		}
   348		return hostname, nil
   349	}
   350	
   351	func setBlobpackedRecovery() {
   352		if *flagRecovery == 0 && env.OnGCE() {
   353			*flagRecovery = gce.BlobpackedRecoveryValue()
   354		}
   355		if blobpacked.RecoveryMode(*flagRecovery) > blobpacked.NoRecovery {
   356			blobpacked.SetRecovery(blobpacked.RecoveryMode(*flagRecovery))
   357		}
   358	}
   359	
   360	// checkGeoKey returns nil if we have a Google Geocoding API key file stored
   361	// in the config dir. Otherwise it returns instruction about it as the error.
   362	func checkGeoKey() error {
   363		if _, err := geocode.GetAPIKey(); err == nil {
   364			return nil
   365		}
   366		keyPath, err := geocode.GetAPIKeyPath()
   367		if err != nil {
   368			return fmt.Errorf("error getting Geocoding API key path: %w", err)
   369		}
   370		if env.OnGCE() {
   371			keyPath = strings.TrimPrefix(keyPath, "/gcs/")
   372			return fmt.Errorf("using OpenStreetMap for location related requests. To use the Google Geocoding API, create a key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ) and save it in your VM's configuration bucket as: %v", keyPath)
   373		}
   374		return fmt.Errorf("using OpenStreetMap for location related requests. To use the Google Geocoding API, create a key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ) and save it in Perkeep's configuration directory as: %v", keyPath)
   375	}
   376	
   377	func main() {
   378		flag.Parse()
   379	
   380		if *flagVersion {
   381			fmt.Fprintf(os.Stderr, "perkeepd version: %s\nGo version: %s (%s/%s)\n",
   382				buildinfo.Summary(), runtime.Version(), runtime.GOOS, runtime.GOARCH)
   383			return
   384		}
   385		if *flagHelp {
   386			flag.Usage()
   387			os.Exit(0)
   388		}
   389		if *flagLegal {
   390			for _, l := range legal.Licenses() {
   391				fmt.Fprintln(os.Stderr, l)
   392			}
   393			return
   394		}
   395		setBlobpackedRecovery()
   396	
   397		// In case we're running in a Docker container with no
   398		// filesystem from which to load the root CAs, this
   399		// conditionally installs a static set if necessary. We do
   400		// this before we load the config file, which might come from
   401		// an https URL. And also before setting up the logging,
   402		// as it uses an http Client.
   403		httputil.InstallCerts()
   404	
   405		logCloser := setupLogging()
   406		defer func() {
   407			if err := logCloser.Close(); err != nil {
   408				log.SetOutput(os.Stderr)
   409				log.Printf("Error closing logger: %v", err)
   410			}
   411		}()
   412	
   413		log.Printf("Starting perkeepd version %s; Go %s (%s/%s)", buildinfo.Summary(), runtime.Version(),
   414			runtime.GOOS, runtime.GOARCH)
   415	
   416		shutdownc := make(chan io.Closer, 1) // receives io.Closer to cleanly shut down
   417		go handleSignals(shutdownc)
   418	
   419		config, err := loadConfig(*flagConfigFile)
   420		if err != nil {
   421			exitf("Error loading config file: %v", err)
   422		}
   423	
   424		ws := webserver.New()
   425		baseURL, err := listen(ws, config)
   426		if err != nil {
   427			exitf("Error starting webserver: %v", err)
   428		}
   429	
   430		config.SetReindex(*flagReindex)
   431		config.SetKeepGoing(*flagKeepGoing)
   432	
   433		// Finally, install the handlers. This also does the final config validation.
   434		shutdownCloser, err := config.InstallHandlers(ws, baseURL)
   435		if err != nil {
   436			exitf("Error parsing config: %v", err)
   437		}
   438		shutdownc <- shutdownCloser
   439	
   440		go ws.Serve()
   441	
   442		if env.OnGCE() {
   443			gce.FixUserDataForPerkeepRename()
   444		}
   445	
   446		if err := checkGeoKey(); err != nil {
   447			log.Printf("perkeepd: %v", err)
   448		}
   449	
   450		urlToOpen := baseURL + config.UIPath()
   451	
   452		if *flagOpenBrowser {
   453			go osutil.OpenURL(urlToOpen)
   454		}
   455	
   456		if flagPollParent {
   457			osutil.DieOnParentDeath()
   458		}
   459	
   460		ctx := context.Background()
   461		if err := config.UploadPublicKey(ctx); err != nil {
   462			exitf("Error uploading public key on startup: %v", err)
   463		}
   464	
   465		if err := config.StartApps(); err != nil {
   466			exitf("StartApps: %v", err)
   467		}
   468	
   469		for appName, appURL := range config.AppURL() {
   470			addr, err := netutil.HostPort(appURL)
   471			if err != nil {
   472				log.Printf("Could not get app %v address: %v", appName, err)
   473				continue
   474			}
   475			if err := netutil.AwaitReachable(addr, 5*time.Second); err != nil {
   476				log.Printf("Could not reach app %v: %v", appName, err)
   477			}
   478		}
   479		log.Printf("server: available at %s", urlToOpen)
   480	
   481		select {}
   482	}