Home Download Docs Code Community
     1	/*
     2	Copyright 2011 The Perkeep Authors.
     3	
     4	Licensed under the Apache License, Version 2.0 (the "License");
     5	you may not use this file except in compliance with the License.
     6	You may obtain a copy of the License at
     7	
     8	     http://www.apache.org/licenses/LICENSE-2.0
     9	
    10	Unless required by applicable law or agreed to in writing, software
    11	distributed under the License is distributed on an "AS IS" BASIS,
    12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13	See the License for the specific language governing permissions and
    14	limitations under the License.
    15	*/
    16	
    17	// Package client implements a Perkeep client.
    18	package client // import "perkeep.org/pkg/client"
    19	
    20	import (
    21		"bytes"
    22		"context"
    23		"crypto/tls"
    24		"encoding/json"
    25		"errors"
    26		"fmt"
    27		"io"
    28		"io/ioutil"
    29		"log"
    30		"net"
    31		"net/http"
    32		"net/url"
    33		"os"
    34		"regexp"
    35		"strings"
    36		"sync"
    37		"time"
    38	
    39		"perkeep.org/internal/hashutil"
    40		"perkeep.org/internal/httputil"
    41		"perkeep.org/internal/osutil"
    42		"perkeep.org/pkg/auth"
    43		"perkeep.org/pkg/blob"
    44		"perkeep.org/pkg/blobserver"
    45		"perkeep.org/pkg/buildinfo"
    46		"perkeep.org/pkg/client/android"
    47		"perkeep.org/pkg/schema"
    48		"perkeep.org/pkg/search"
    49		"perkeep.org/pkg/types/camtypes"
    50	
    51		"go4.org/syncutil"
    52		"golang.org/x/net/http2"
    53	)
    54	
    55	// A Client provides access to a Perkeep server.
    56	//
    57	// After use, a Client should be closed via its Close method to
    58	// release idle HTTP connections or other resourcedds.
    59	type Client struct {
    60		// server is the input from user, pre-discovery.
    61		// For example "http://foo.com" or "foo.com:1234".
    62		// It is the responsibility of initPrefix to parse
    63		// server and set prefix, including doing discovery
    64		// to figure out what the proper server-declared
    65		// prefix is.
    66		server string
    67	
    68		prefixOnce    syncutil.Once // guards init of following 2 fields
    69		prefixv       string        // URL prefix before "/camli/"
    70		isSharePrefix bool          // URL is a request for a share blob
    71	
    72		discoOnce              syncutil.Once
    73		searchRoot             string      // Handler prefix, or "" if none
    74		downloadHelper         string      // or "" if none
    75		storageGen             string      // storage generation, or "" if not reported
    76		hasLegacySHA1          bool        // Whether server has SHA-1 blobs indexed.
    77		syncHandlers           []*SyncInfo // "from" and "to" url prefix for each syncHandler
    78		serverKeyID            string      // Server's GPG public key ID.
    79		helpRoot               string      // Handler prefix, or "" if none
    80		shareRoot              string      // Share handler prefix, or "" if none
    81		serverPublicKeyBlobRef blob.Ref    // Server's public key blobRef
    82	
    83		signerOnce  sync.Once
    84		signer      *schema.Signer
    85		signerErr   error
    86		signHandler string // Handler prefix, or "" if none
    87	
    88		authMode auth.AuthMode
    89		// authErr is set when no auth config is found but we want to defer warning
    90		// until discovery fails.
    91		authErr error
    92	
    93		httpClient *http.Client
    94		haveCache  HaveCache
    95	
    96		// If sto is set, it's used before the httpClient or other network operations.
    97		sto blobserver.Storage
    98	
    99		initTrustedCertsOnce sync.Once
   100	
   101		// We define a certificate fingerprint as the 20 digits lowercase prefix
   102		// of the SHA256 of the complete certificate (in ASN.1 DER encoding).
   103		// trustedCerts contains the fingerprints of the self-signed
   104		// certificates we trust.
   105		// If not empty, (and if using TLS) the full x509 verification is
   106		// disabled, and we instead check the server's certificate against
   107		// this list.
   108		// The perkeepd server prints the fingerprint to add to the config
   109		// when starting.
   110		trustedCerts []string
   111	
   112		// insecureAnyTLSCert disables all TLS cert checking,
   113		// including the trustedCerts field above.
   114		insecureAnyTLSCert bool
   115	
   116		initIgnoredFilesOnce sync.Once
   117		// list of files that pk-put should ignore.
   118		// Defaults to empty, but pk-put init creates a config with a non
   119		// empty list.
   120		// See IsIgnoredFile for the matching rules.
   121		ignoredFiles  []string
   122		ignoreChecker func(path string) bool
   123	
   124		initSignerPublicKeyBlobrefOnce sync.Once
   125		signerPublicKeyRef             blob.Ref
   126		publicKeyArmored               string
   127	
   128		statsMutex sync.Mutex
   129		stats      Stats
   130	
   131		// via maps the access path from a share root to a desired target.
   132		// It is non-nil when in "sharing" mode, where the Client is fetching
   133		// a share.
   134		viaMu sync.RWMutex
   135		via   map[blob.Ref]blob.Ref // target => via (target is referenced from via)
   136	
   137		// Verbose controls how much logging from the client is printed. The caller
   138		// should set it only before using the client, and treat it as read-only after
   139		// that.
   140		Verbose bool
   141		// Logger is the logger used by the client. It defaults to a standard
   142		// logger to os.Stderr if the client is initialized by one of the package's
   143		// functions. Like Verbose, it should be set only before using the client, and
   144		// not be modified afterwards.
   145		Logger *log.Logger
   146	
   147		httpGate        *syncutil.Gate
   148		transportConfig *TransportConfig // or nil
   149	
   150		noExtConfig bool // no external config; config file and env vars are ignored.
   151	
   152		// sameOrigin indicates whether URLs in requests should be stripped from
   153		// their scheme and HostPort parts. This is meant for when using the client
   154		// through gopherjs in the web UI. Because we'll run into CORS errors if
   155		// requests have a Host part.
   156		sameOrigin bool
   157	}
   158	
   159	const maxParallelHTTP_h1 = 5
   160	const maxParallelHTTP_h2 = 50
   161	
   162	// inGopherJS reports whether the client package is compiled by GopherJS, for use
   163	// in the browser.
   164	var inGopherJS bool
   165	
   166	// New returns a new Perkeep Client.
   167	//
   168	// By default, with no options, it uses the client as configured in
   169	// the environment or default configuration files.
   170	func New(opts ...ClientOption) (*Client, error) {
   171		c := &Client{
   172			haveCache: noHaveCache{},
   173			Logger:    log.New(os.Stderr, "", log.Ldate|log.Ltime),
   174			authMode:  auth.None{},
   175		}
   176		for _, v := range opts {
   177			v.modifyClient(c)
   178		}
   179		if c.sto != nil && len(opts) > 1 {
   180			return nil, errors.New("use of OptionUseStorageClient precludes use of any other options")
   181		}
   182	
   183		if inGopherJS {
   184			c.noExtConfig = true
   185			c.sameOrigin = true
   186		}
   187		if c.noExtConfig {
   188			c.setDefaultHTTPClient()
   189			return c, nil
   190		}
   191	
   192		if c.server != "" {
   193			if !isURLOrHostPort(c.server) {
   194				configOnce.Do(parseConfig)
   195				serverConf, ok := config.Servers[c.server]
   196				if !ok {
   197					log.Fatalf("%q looks like a server alias, but no such alias found in config at %v", c.server, osutil.UserClientConfigPath())
   198				}
   199				c.server = serverConf.Server
   200			}
   201			c.setDefaultHTTPClient()
   202			return c, nil
   203		}
   204	
   205		var err error
   206		c.server, err = getServer()
   207		if err != nil {
   208			return nil, err
   209		}
   210		err = c.SetupAuth()
   211		if err != nil {
   212			return nil, err
   213		}
   214		c.setDefaultHTTPClient()
   215		return c, nil
   216	}
   217	
   218	func (c *Client) setDefaultHTTPClient() {
   219		if c.httpClient == nil {
   220			c.httpClient = &http.Client{
   221				Transport: c.transportForConfig(c.transportConfig),
   222			}
   223		}
   224		c.httpGate = syncutil.NewGate(httpGateSize(c.httpClient.Transport))
   225	}
   226	
   227	// NewOrFail is like New, but calls log.Fatal instead of returning an error.
   228	func NewOrFail(opts ...ClientOption) *Client {
   229		c, err := New(opts...)
   230		if err != nil {
   231			log.Fatalf("error creating client: %v", err)
   232		}
   233		return c
   234	}
   235	
   236	// NewPathClient returns a new client accessing a subpath of c.
   237	func (c *Client) NewPathClient(path string) (*Client, error) {
   238		u, err := url.Parse(c.server)
   239		if err != nil {
   240			return nil, fmt.Errorf("bogus server %q for NewPathClient receiver: %v", c.server, err)
   241		}
   242		u.Path = path
   243		pc, err := New(OptionServer(u.String()))
   244		if err != nil {
   245			return nil, err
   246		}
   247		pc.authMode = c.authMode
   248		pc.discoOnce.Do(noop)
   249		return pc, nil
   250	}
   251	
   252	// TransportConfig contains options for how HTTP requests are made.
   253	type TransportConfig struct {
   254		// Proxy optionally specifies the Proxy for the transport. Useful with
   255		// pk-put for debugging even localhost requests.
   256		Proxy   func(*http.Request) (*url.URL, error)
   257		Verbose bool // Verbose enables verbose logging of HTTP requests.
   258	}
   259	
   260	func (c *Client) useHTTP2(tc *TransportConfig) bool {
   261		if !c.useTLS() {
   262			return false
   263		}
   264		if android.IsChild() {
   265			// No particular reason; just untested so far.
   266			return false
   267		}
   268		if os.Getenv("HTTPS_PROXY") != "" || os.Getenv("https_proxy") != "" ||
   269			(tc != nil && tc.Proxy != nil) {
   270			// Also just untested. Which proxies support h2 anyway?
   271			return false
   272		}
   273		return true
   274	}
   275	
   276	// transportForConfig returns a transport for the client, setting the correct
   277	// Proxy, Dial, and TLSClientConfig if needed. It does not mutate c.
   278	// It is the caller's responsibility to then use that transport to set
   279	// the client's httpClient with SetHTTPClient.
   280	func (c *Client) transportForConfig(tc *TransportConfig) http.RoundTripper {
   281		if inGopherJS {
   282			// Calls to net.Dial* functions - which would happen if the client's transport
   283			// is not nil - are prohibited with GopherJS. So we force nil here, so that the
   284			// call to transportForConfig in newClient is of no consequence when on the
   285			// browser.
   286			return nil
   287		}
   288		if c == nil {
   289			return nil
   290		}
   291		var transport http.RoundTripper
   292		proxy := http.ProxyFromEnvironment
   293		if tc != nil && tc.Proxy != nil {
   294			proxy = tc.Proxy
   295		}
   296	
   297		if c.useHTTP2(tc) {
   298			transport = &http2.Transport{
   299				DialTLS: c.http2DialTLSFunc(),
   300			}
   301		} else {
   302			transport = &http.Transport{
   303				DialTLS:             c.DialTLSFunc(),
   304				Dial:                c.DialFunc(),
   305				Proxy:               proxy,
   306				MaxIdleConnsPerHost: maxParallelHTTP_h1,
   307			}
   308		}
   309		httpStats := &httputil.StatsTransport{
   310			Transport: transport,
   311		}
   312		if tc != nil {
   313			httpStats.VerboseLog = tc.Verbose
   314		}
   315		transport = httpStats
   316		if android.IsChild() {
   317			transport = &android.StatsTransport{Rt: transport}
   318		}
   319		return transport
   320	}
   321	
   322	// HTTPStats returns the client's underlying httputil.StatsTransport, if in use.
   323	// If another transport is being used, nil is returned.
   324	func (c *Client) HTTPStats() *httputil.StatsTransport {
   325		st, _ := c.httpClient.Transport.(*httputil.StatsTransport)
   326		return st
   327	}
   328	
   329	type ClientOption interface {
   330		modifyClient(*Client)
   331	}
   332	
   333	// OptionServer returns a Client constructor option that forces use of
   334	// the provided server.
   335	//
   336	// The provided server is either "host:port" (assumed http, not https)
   337	// or a URL prefix, with or without a path, or a server alias from the
   338	// client configuration file. A server alias should not be confused
   339	// with a hostname, therefore it cannot contain any colon or period.
   340	func OptionServer(server string) ClientOption {
   341		return optionServer(server)
   342	}
   343	
   344	type optionServer string
   345	
   346	func (s optionServer) modifyClient(c *Client) { c.server = string(s) }
   347	
   348	// OptionUseStorageClient returns a Client constructor option that
   349	// forces use of the provided blob storage target.
   350	//
   351	// This exists mainly so all the convenience methods on
   352	// Client (e.g. the Upload variants) are available against storage
   353	// directly.
   354	//
   355	// Use of OptionUseStorageClient is mutually exclusively
   356	// with all other options, although it does imply
   357	// OptionNoExternalConfig(true).
   358	func OptionUseStorageClient(s blobserver.Storage) ClientOption {
   359		return optionStorage{s}
   360	}
   361	
   362	type optionStorage struct {
   363		sto blobserver.Storage
   364	}
   365	
   366	func (o optionStorage) modifyClient(c *Client) {
   367		c.sto = o.sto
   368		c.noExtConfig = true
   369	}
   370	
   371	// OptionTransportConfig returns a ClientOption that makes the client use
   372	// the provided transport configuration options.
   373	func OptionTransportConfig(tc *TransportConfig) ClientOption {
   374		return optionTransportConfig{tc}
   375	}
   376	
   377	type optionTransportConfig struct {
   378		tc *TransportConfig
   379	}
   380	
   381	func (o optionTransportConfig) modifyClient(c *Client) { c.transportConfig = o.tc }
   382	
   383	// OptionInsecure returns a ClientOption that controls whether HTTP
   384	// requests are allowed to be insecure (over HTTP or HTTPS without TLS
   385	// certificate checking). Use of this is strongly discouraged except
   386	// for local testing.
   387	func OptionInsecure(v bool) ClientOption {
   388		return optionInsecure(v)
   389	}
   390	
   391	type optionInsecure bool
   392	
   393	func (o optionInsecure) modifyClient(c *Client) { c.insecureAnyTLSCert = bool(o) }
   394	
   395	// OptionTrustedCert returns a ClientOption that makes the client
   396	// trust the provide self-signed cert signature. The value should be
   397	// the 20 byte hex prefix of the SHA-256 of the cert, as printed by
   398	// the perkeepd server on start-up.
   399	//
   400	// If cert is empty, the option has no effect.
   401	func OptionTrustedCert(cert string) ClientOption {
   402		// TODO: remove this whole function now that we have LetsEncrypt?
   403		return optionTrustedCert(cert)
   404	}
   405	
   406	type optionTrustedCert string
   407	
   408	func (o optionTrustedCert) modifyClient(c *Client) {
   409		cert := string(o)
   410		if cert != "" {
   411			c.initTrustedCertsOnce.Do(func() {})
   412			c.trustedCerts = []string{string(o)}
   413		}
   414	}
   415	
   416	type optionNoExtConfig bool
   417	
   418	func (o optionNoExtConfig) modifyClient(c *Client) { c.noExtConfig = bool(o) }
   419	
   420	// OptionNoExternalConfig returns a Client constructor option that
   421	// prevents any on-disk config files or environment variables from
   422	// influencing the client. It may still use the disk for caches.
   423	func OptionNoExternalConfig() ClientOption {
   424		return optionNoExtConfig(true)
   425	}
   426	
   427	type optionAuthMode struct {
   428		m auth.AuthMode
   429	}
   430	
   431	func (o optionAuthMode) modifyClient(c *Client) { c.authMode = o.m }
   432	
   433	// OptionAuthMode returns a Client constructor option that sets the
   434	// client's authentication mode.
   435	func OptionAuthMode(m auth.AuthMode) ClientOption {
   436		return optionAuthMode{m}
   437	}
   438	
   439	// noop is for use with syncutil.Onces.
   440	func noop() error { return nil }
   441	
   442	var shareURLRx = regexp.MustCompile(`^(.+)/(` + blob.Pattern + ")$")
   443	
   444	// NewFromShareRoot uses shareBlobURL to set up and return a client that
   445	// will be used to fetch shared blobs.
   446	func NewFromShareRoot(ctx context.Context, shareBlobURL string, opts ...ClientOption) (c *Client, target blob.Ref, err error) {
   447		var root string
   448		m := shareURLRx.FindStringSubmatch(shareBlobURL)
   449		if m == nil {
   450			return nil, blob.Ref{}, fmt.Errorf("Unknown share URL base")
   451		}
   452		c, err = New(append(opts[:len(opts):cap(opts)], OptionServer(m[1]))...)
   453		if err != nil {
   454			return nil, blob.Ref{}, err
   455		}
   456		c.discoOnce.Do(noop)
   457		c.prefixOnce.Do(noop)
   458		c.prefixv = m[1]
   459		c.isSharePrefix = true
   460		c.authMode = auth.None{}
   461		c.via = make(map[blob.Ref]blob.Ref)
   462		root = m[2]
   463	
   464		req := c.newRequest(ctx, "GET", shareBlobURL, nil)
   465		res, err := c.expect2XX(req)
   466		if err != nil {
   467			return nil, blob.Ref{}, fmt.Errorf("error fetching %s: %v", shareBlobURL, err)
   468		}
   469		defer res.Body.Close()
   470		var buf bytes.Buffer
   471		rootbr, ok := blob.Parse(root)
   472		if !ok {
   473			return nil, blob.Ref{}, fmt.Errorf("invalid root blob ref for sharing: %q", root)
   474		}
   475		b, err := schema.BlobFromReader(rootbr, io.TeeReader(res.Body, &buf))
   476		if err != nil {
   477			return nil, blob.Ref{}, fmt.Errorf("error parsing JSON from %s: %v , with response: %q", shareBlobURL, err, buf.Bytes())
   478		}
   479		if b.ShareAuthType() != schema.ShareHaveRef {
   480			return nil, blob.Ref{}, fmt.Errorf("unknown share authType of %q", b.ShareAuthType())
   481		}
   482		target = b.ShareTarget()
   483		if !target.Valid() {
   484			return nil, blob.Ref{}, fmt.Errorf("no target")
   485		}
   486		c.via[target] = rootbr
   487		return c, target, nil
   488	}
   489	
   490	// SetHTTPClient sets the Perkeep client's HTTP client.
   491	// If nil, the default HTTP client is used.
   492	func (c *Client) SetHTTPClient(client *http.Client) {
   493		if client == nil {
   494			client = http.DefaultClient
   495		}
   496		c.httpClient = client
   497	}
   498	
   499	// HTTPClient returns the Client's underlying http.Client.
   500	func (c *Client) HTTPClient() *http.Client {
   501		return c.httpClient
   502	}
   503	
   504	// A HaveCache caches whether a remote blobserver has a blob.
   505	type HaveCache interface {
   506		StatBlobCache(br blob.Ref) (size uint32, ok bool)
   507		NoteBlobExists(br blob.Ref, size uint32)
   508	}
   509	
   510	type noHaveCache struct{}
   511	
   512	func (noHaveCache) StatBlobCache(blob.Ref) (uint32, bool) { return 0, false }
   513	func (noHaveCache) NoteBlobExists(blob.Ref, uint32)       {}
   514	
   515	func (c *Client) SetHaveCache(cache HaveCache) {
   516		if cache == nil {
   517			cache = noHaveCache{}
   518		}
   519		c.haveCache = cache
   520	}
   521	
   522	func (c *Client) printf(format string, v ...interface{}) {
   523		if c.Verbose && c.Logger != nil {
   524			c.Logger.Printf(format, v...)
   525		}
   526	}
   527	
   528	func (c *Client) Stats() Stats {
   529		c.statsMutex.Lock()
   530		defer c.statsMutex.Unlock()
   531		return c.stats // copy
   532	}
   533	
   534	// ErrNoSearchRoot is returned by SearchRoot if the server doesn't support search.
   535	var ErrNoSearchRoot = errors.New("client: server doesn't support search")
   536	
   537	// ErrNoHelpRoot is returned by HelpRoot if the server doesn't have a help handler.
   538	var ErrNoHelpRoot = errors.New("client: server does not have a help handler")
   539	
   540	// ErrNoShareRoot is returned by ShareRoot if the server doesn't have a share handler.
   541	var ErrNoShareRoot = errors.New("client: server does not have a share handler")
   542	
   543	// ErrNoSigning is returned by ServerKeyID if the server doesn't support signing.
   544	var ErrNoSigning = fmt.Errorf("client: server doesn't support signing")
   545	
   546	// ErrNoStorageGeneration is returned by StorageGeneration if the
   547	// server doesn't report a storage generation value.
   548	var ErrNoStorageGeneration = errors.New("client: server doesn't report a storage generation")
   549	
   550	// ErrNoSync is returned by SyncHandlers if the server does not advertise syncs.
   551	var ErrNoSync = errors.New("client: server has no sync handlers")
   552	
   553	// BlobRoot returns the server's blobroot URL prefix.
   554	// If the client was constructed with an explicit path,
   555	// that path is used. Otherwise the server's
   556	// default advertised blobRoot is used.
   557	func (c *Client) BlobRoot() (string, error) {
   558		prefix, err := c.prefix()
   559		if err != nil {
   560			return "", err
   561		}
   562		return prefix + "/", nil
   563	}
   564	
   565	// ServerKeyID returns the server's GPG public key ID, in its long (16 capital
   566	// hex digits) format. If the server isn't running a sign handler, the error
   567	// will be ErrNoSigning.
   568	func (c *Client) ServerKeyID() (string, error) {
   569		if err := c.condDiscovery(); err != nil {
   570			return "", err
   571		}
   572		if c.serverKeyID == "" {
   573			return "", ErrNoSigning
   574		}
   575		return c.serverKeyID, nil
   576	}
   577	
   578	// ServerPublicKeyBlobRef returns the server's public key blobRef
   579	// If the server isn't running a sign handler, the error will be ErrNoSigning.
   580	func (c *Client) ServerPublicKeyBlobRef() (blob.Ref, error) {
   581		if err := c.condDiscovery(); err != nil {
   582			return blob.Ref{}, err
   583		}
   584	
   585		if !c.serverPublicKeyBlobRef.Valid() {
   586			return blob.Ref{}, ErrNoSigning
   587		}
   588		return c.serverPublicKeyBlobRef, nil
   589	}
   590	
   591	// SearchRoot returns the server's search handler.
   592	// If the server isn't running an index and search handler, the error
   593	// will be ErrNoSearchRoot.
   594	func (c *Client) SearchRoot() (string, error) {
   595		if err := c.condDiscovery(); err != nil {
   596			return "", err
   597		}
   598		if c.searchRoot == "" {
   599			return "", ErrNoSearchRoot
   600		}
   601		return c.searchRoot, nil
   602	}
   603	
   604	// HelpRoot returns the server's help handler.
   605	// If the server isn't running a help handler, the error will be
   606	// ErrNoHelpRoot.
   607	func (c *Client) HelpRoot() (string, error) {
   608		if err := c.condDiscovery(); err != nil {
   609			return "", err
   610		}
   611		if c.helpRoot == "" {
   612			return "", ErrNoHelpRoot
   613		}
   614		return c.helpRoot, nil
   615	}
   616	
   617	// ShareRoot returns the server's share handler prefix URL.
   618	// If the server isn't running a share handler, the error will be
   619	// ErrNoShareRoot.
   620	func (c *Client) ShareRoot() (string, error) {
   621		if err := c.condDiscovery(); err != nil {
   622			return "", err
   623		}
   624		if c.shareRoot == "" {
   625			return "", ErrNoShareRoot
   626		}
   627		return c.shareRoot, nil
   628	}
   629	
   630	// SignHandler returns the server's sign handler.
   631	// If the server isn't running a sign handler, the error will be
   632	// ErrNoSigning.
   633	func (c *Client) SignHandler() (string, error) {
   634		if err := c.condDiscovery(); err != nil {
   635			return "", err
   636		}
   637		if c.signHandler == "" {
   638			return "", ErrNoSigning
   639		}
   640		return c.signHandler, nil
   641	}
   642	
   643	// StorageGeneration returns the server's unique ID for its storage
   644	// generation, reset whenever storage is reset, moved, or partially
   645	// lost.
   646	//
   647	// This is a value that can be used in client cache keys to add
   648	// certainty that they're talking to the same instance as previously.
   649	//
   650	// If the server doesn't return such a value, the error will be
   651	// ErrNoStorageGeneration.
   652	func (c *Client) StorageGeneration() (string, error) {
   653		if err := c.condDiscovery(); err != nil {
   654			return "", err
   655		}
   656		if c.storageGen == "" {
   657			return "", ErrNoStorageGeneration
   658		}
   659		return c.storageGen, nil
   660	}
   661	
   662	// HasLegacySHA1 reports whether the server has SHA-1 blobs indexed.
   663	func (c *Client) HasLegacySHA1() (bool, error) {
   664		if err := c.condDiscovery(); err != nil {
   665			return false, err
   666		}
   667		return c.hasLegacySHA1, nil
   668	}
   669	
   670	// SyncInfo holds the data that were acquired with a discovery
   671	// and that are relevant to a syncHandler.
   672	type SyncInfo struct {
   673		From    string
   674		To      string
   675		ToIndex bool // whether this sync is from a blob storage to an index
   676	}
   677	
   678	// SyncHandlers returns the server's sync handlers "from" and
   679	// "to" prefix URLs.
   680	// If the server isn't running any sync handler, the error
   681	// will be ErrNoSync.
   682	func (c *Client) SyncHandlers() ([]*SyncInfo, error) {
   683		if err := c.condDiscovery(); err != nil {
   684			return nil, err
   685		}
   686		if c.syncHandlers == nil {
   687			return nil, ErrNoSync
   688		}
   689		return c.syncHandlers, nil
   690	}
   691	
   692	var _ search.GetRecentPermanoder = (*Client)(nil)
   693	
   694	// GetRecentPermanodes implements search.GetRecentPermanoder against a remote server over HTTP.
   695	func (c *Client) GetRecentPermanodes(ctx context.Context, req *search.RecentRequest) (*search.RecentResponse, error) {
   696		sr, err := c.SearchRoot()
   697		if err != nil {
   698			return nil, err
   699		}
   700		url := sr + req.URLSuffix()
   701		hreq := c.newRequest(ctx, "GET", url)
   702		hres, err := c.expect2XX(hreq)
   703		if err != nil {
   704			return nil, err
   705		}
   706		res := new(search.RecentResponse)
   707		if err := httputil.DecodeJSON(hres, res); err != nil {
   708			return nil, err
   709		}
   710		if err := res.Err(); err != nil {
   711			return nil, err
   712		}
   713		return res, nil
   714	}
   715	
   716	// GetPermanodesWithAttr searches for permanodes that match the given search request.
   717	// The Fuzzy option in the request must not be set, and the Attribute option
   718	// must be set.
   719	// Only indexed attributes may be queried.
   720	func (c *Client) GetPermanodesWithAttr(ctx context.Context, req *search.WithAttrRequest) (*search.WithAttrResponse, error) {
   721		sr, err := c.SearchRoot()
   722		if err != nil {
   723			return nil, err
   724		}
   725		url := sr + req.URLSuffix()
   726		hreq := c.newRequest(ctx, "GET", url)
   727		hres, err := c.expect2XX(hreq)
   728		if err != nil {
   729			return nil, err
   730		}
   731		res := new(search.WithAttrResponse)
   732		if err := httputil.DecodeJSON(hres, res); err != nil {
   733			return nil, err
   734		}
   735		if err := res.Err(); err != nil {
   736			return nil, err
   737		}
   738		return res, nil
   739	}
   740	
   741	func (c *Client) Describe(ctx context.Context, req *search.DescribeRequest) (*search.DescribeResponse, error) {
   742		sr, err := c.SearchRoot()
   743		if err != nil {
   744			return nil, err
   745		}
   746		url := sr + req.URLSuffixPost()
   747		body, err := json.MarshalIndent(req, "", "\t")
   748		if err != nil {
   749			return nil, err
   750		}
   751		hreq := c.newRequest(ctx, "POST", url, bytes.NewReader(body))
   752		hres, err := c.expect2XX(hreq)
   753		if err != nil {
   754			return nil, err
   755		}
   756		res := new(search.DescribeResponse)
   757		if err := httputil.DecodeJSON(hres, res); err != nil {
   758			return nil, err
   759		}
   760		return res, nil
   761	}
   762	
   763	func (c *Client) GetClaims(ctx context.Context, req *search.ClaimsRequest) (*search.ClaimsResponse, error) {
   764		sr, err := c.SearchRoot()
   765		if err != nil {
   766			return nil, err
   767		}
   768		url := sr + req.URLSuffix()
   769		hreq := c.newRequest(ctx, "GET", url)
   770		hres, err := c.expect2XX(hreq)
   771		if err != nil {
   772			return nil, err
   773		}
   774		res := new(search.ClaimsResponse)
   775		if err := httputil.DecodeJSON(hres, res); err != nil {
   776			return nil, err
   777		}
   778		return res, nil
   779	}
   780	
   781	func (c *Client) query(ctx context.Context, req *search.SearchQuery) (*http.Response, error) {
   782		sr, err := c.SearchRoot()
   783		if err != nil {
   784			return nil, err
   785		}
   786		url := sr + req.URLSuffix()
   787		body, err := json.Marshal(req)
   788		if err != nil {
   789			return nil, err
   790		}
   791		hreq := c.newRequest(ctx, "POST", url, bytes.NewReader(body))
   792		return c.expect2XX(hreq)
   793	}
   794	
   795	func (c *Client) Query(ctx context.Context, req *search.SearchQuery) (*search.SearchResult, error) {
   796		hres, err := c.query(ctx, req)
   797		if err != nil {
   798			return nil, err
   799		}
   800		res := new(search.SearchResult)
   801		if err := httputil.DecodeJSON(hres, res); err != nil {
   802			return nil, err
   803		}
   804		return res, nil
   805	}
   806	
   807	// QueryRaw sends req and returns the body of the response, which should be the
   808	// unparsed JSON of a search.SearchResult.
   809	func (c *Client) QueryRaw(ctx context.Context, req *search.SearchQuery) ([]byte, error) {
   810		hres, err := c.query(ctx, req)
   811		if err != nil {
   812			return nil, err
   813		}
   814		defer hres.Body.Close()
   815		return ioutil.ReadAll(hres.Body)
   816	}
   817	
   818	// SearchExistingFileSchema does a search query looking for an
   819	// existing file with entire contents of wholeRef, then does a HEAD
   820	// request to verify the file still exists on the server. If so,
   821	// it returns that file schema's blobref.
   822	//
   823	// If multiple wholeRef values are provided, any may match. This is
   824	// used for searching for the file by multiple wholeRef hashes
   825	// (e.g. SHA-224 and SHA-1).
   826	//
   827	// It returns (zero, nil) if not found. A non-nil error is only returned
   828	// if there were problems searching.
   829	func (c *Client) SearchExistingFileSchema(ctx context.Context, wholeRef ...blob.Ref) (blob.Ref, error) {
   830		sr, err := c.SearchRoot()
   831		if err != nil {
   832			return blob.Ref{}, err
   833		}
   834		if len(wholeRef) == 0 {
   835			return blob.Ref{}, nil
   836		}
   837		url := sr + "camli/search/files"
   838		for i, ref := range wholeRef {
   839			if i == 0 {
   840				url += "?wholedigest=" + ref.String()
   841			} else {
   842				url += "&wholedigest=" + ref.String()
   843			}
   844		}
   845		req := c.newRequest(ctx, "GET", url)
   846		res, err := c.doReqGated(req)
   847		if err != nil {
   848			return blob.Ref{}, err
   849		}
   850		if res.StatusCode != 200 {
   851			body, _ := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20))
   852			res.Body.Close()
   853			return blob.Ref{}, fmt.Errorf("client: got status code %d from URL %s; body %s", res.StatusCode, url, body)
   854		}
   855		var ress camtypes.FileSearchResponse
   856		if err := httputil.DecodeJSON(res, &ress); err != nil {
   857			// Check that we're not just hitting the change introduced in 2018-01-13-6e8a5930c9fee81640c6c75a9a549fec98064186
   858			mismatch, err := c.versionMismatch(ctx)
   859			if err != nil {
   860				log.Printf("Could not verify whether client is too recent or server is too old: %v", err)
   861			} else if mismatch {
   862				return blob.Ref{}, fmt.Errorf("Client is too recent for this server. Use a client built before 2018-01-13-6e8a5930c9, or upgrade the server to after that revision.")
   863			}
   864			return blob.Ref{}, fmt.Errorf("client: error parsing JSON from URL %s: %v", url, err)
   865		}
   866		if len(ress.Files) == 0 {
   867			return blob.Ref{}, nil
   868		}
   869		for wholeRef, files := range ress.Files {
   870			for _, f := range files {
   871				if c.FileHasContents(ctx, f, blob.MustParse(wholeRef)) {
   872					return f, nil
   873				}
   874			}
   875		}
   876		return blob.Ref{}, nil
   877	}
   878	
   879	// versionMismatch returns true if the server was built before 2018-01-13 and
   880	// the client was built at or after 2018-01-13.
   881	func (c *Client) versionMismatch(ctx context.Context) (bool, error) {
   882		const shortRFC3339 = "2006-01-02"
   883		version := buildinfo.GitInfo
   884		if version == "" {
   885			return false, errors.New("unknown client version")
   886		}
   887		version = version[:10] // keep only the date part
   888		clientDate, err := time.Parse(shortRFC3339, version)
   889		if err != nil {
   890			return false, fmt.Errorf("could not parse date from version %q: %v", version, err)
   891		}
   892		apiChangeDate, _ := time.Parse(shortRFC3339, "2018-01-13")
   893		if !clientDate.After(apiChangeDate) {
   894			// client is old enough, all good.
   895			return false, nil
   896		}
   897		url := c.discoRoot() + "/status/status.json"
   898		req := c.newRequest(ctx, "GET", url)
   899		res, err := c.doReqGated(req)
   900		if err != nil {
   901			return false, err
   902		}
   903		if res.StatusCode != 200 {
   904			body, _ := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20))
   905			res.Body.Close()
   906			return false, fmt.Errorf("got status code %d from URL %s; body %s", res.StatusCode, url, body)
   907		}
   908		var status struct {
   909			Version string `json:"version"`
   910		}
   911		if err := httputil.DecodeJSON(res, &status); err != nil {
   912			return false, fmt.Errorf("error parsing JSON from URL %s: %v", url, err)
   913		}
   914		serverVersion := status.Version[:10]
   915		serverDate, err := time.Parse(shortRFC3339, serverVersion)
   916		if err != nil {
   917			return false, fmt.Errorf("could not parse date from server version %q: %v", status.Version, err)
   918		}
   919		if serverDate.After(apiChangeDate) {
   920			// server is recent enough, all good.
   921			return false, nil
   922		}
   923		return true, nil
   924	}
   925	
   926	// FileHasContents returns true iff f refers to a "file" or "bytes" schema blob,
   927	// the server is configured with a "download helper", and the server responds
   928	// that all chunks of 'f' are available and match the digest of wholeRef.
   929	func (c *Client) FileHasContents(ctx context.Context, f, wholeRef blob.Ref) bool {
   930		if err := c.condDiscovery(); err != nil {
   931			return false
   932		}
   933		if c.downloadHelper == "" {
   934			return false
   935		}
   936		req := c.newRequest(ctx, "HEAD", c.downloadHelper+f.String()+"/?verifycontents="+wholeRef.String())
   937		res, err := c.expect2XX(req)
   938		if err != nil {
   939			log.Printf("download helper HEAD error: %v", err)
   940			return false
   941		}
   942		defer res.Body.Close()
   943		return res.Header.Get("X-Camli-Contents") == wholeRef.String()
   944	}
   945	
   946	// prefix returns the URL prefix before "/camli/", or before
   947	// the blobref hash in case of a share URL.
   948	// Examples: http://foo.com:3179/bs or http://foo.com:3179/share
   949	func (c *Client) prefix() (string, error) {
   950		if err := c.prefixOnce.Do(c.initPrefix); err != nil {
   951			return "", err
   952		}
   953		return c.prefixv, nil
   954	}
   955	
   956	// blobPrefix returns the URL prefix before the blobref hash.
   957	// Example: http://foo.com:3179/bs/camli or http://foo.com:3179/share
   958	func (c *Client) blobPrefix() (string, error) {
   959		pfx, err := c.prefix()
   960		if err != nil {
   961			return "", err
   962		}
   963		if !c.isSharePrefix {
   964			pfx += "/camli"
   965		}
   966		return pfx, nil
   967	}
   968	
   969	// discoRoot returns the user defined server for this client. It prepends "https://" if no scheme was specified.
   970	func (c *Client) discoRoot() string {
   971		s := c.server
   972		if c.sameOrigin {
   973			s = strings.TrimPrefix(s, "http://")
   974			s = strings.TrimPrefix(s, "https://")
   975			parts := strings.SplitN(s, "/", 1)
   976			if len(parts) < 2 {
   977				return "/"
   978			}
   979			return "/" + parts[1]
   980		}
   981		if !strings.HasPrefix(s, "http") {
   982			s = "https://" + s
   983		}
   984		return s
   985	}
   986	
   987	// initPrefix uses the user provided server URL to define the URL
   988	// prefix to the blobserver root. If the server URL has a path
   989	// component then it is directly used, otherwise the blobRoot
   990	// from the discovery is used as the path.
   991	func (c *Client) initPrefix() error {
   992		c.isSharePrefix = false
   993		root := c.discoRoot()
   994		u, err := url.Parse(root)
   995		if err != nil {
   996			return err
   997		}
   998		if len(u.Path) > 1 {
   999			c.prefixv = strings.TrimRight(root, "/")
  1000			return nil
  1001		}
  1002		return c.condDiscovery()
  1003	}
  1004	
  1005	func (c *Client) condDiscovery() error {
  1006		if c.sto != nil {
  1007			return errors.New("client not using HTTP")
  1008		}
  1009		return c.discoOnce.Do(c.doDiscovery)
  1010	}
  1011	
  1012	// DiscoveryDoc returns the server's JSON discovery document.
  1013	// This method exists purely for the "camtool discovery" command.
  1014	// Clients shouldn't have to parse this themselves.
  1015	func (c *Client) DiscoveryDoc(ctx context.Context) (io.Reader, error) {
  1016		res, err := c.discoveryResp(ctx)
  1017		if err != nil {
  1018			return nil, err
  1019		}
  1020		defer res.Body.Close()
  1021		const maxSize = 1 << 20
  1022		all, err := ioutil.ReadAll(io.LimitReader(res.Body, maxSize+1))
  1023		if err != nil {
  1024			return nil, err
  1025		}
  1026		if len(all) > maxSize {
  1027			return nil, errors.New("discovery document oddly large")
  1028		}
  1029		if len(all) > 0 && all[len(all)-1] != '\n' {
  1030			all = append(all, '\n')
  1031		}
  1032		return bytes.NewReader(all), err
  1033	}
  1034	
  1035	// HTTPVersion reports the HTTP version in use, such as "HTTP/1.1" or "HTTP/2.0".
  1036	func (c *Client) HTTPVersion(ctx context.Context) (string, error) {
  1037		req := c.newRequest(ctx, "HEAD", c.discoRoot(), nil)
  1038		res, err := c.doReqGated(req)
  1039		if err != nil {
  1040			return "", err
  1041		}
  1042		return res.Proto, err
  1043	}
  1044	
  1045	func (c *Client) discoveryResp(ctx context.Context) (*http.Response, error) {
  1046		// If the path is just "" or "/", do discovery against
  1047		// the URL to see which path we should actually use.
  1048		req := c.newRequest(ctx, "GET", c.discoRoot(), nil)
  1049		req.Header.Set("Accept", "text/x-camli-configuration")
  1050		res, err := c.doReqGated(req)
  1051		if err != nil {
  1052			return nil, err
  1053		}
  1054		if res.StatusCode != 200 {
  1055			res.Body.Close()
  1056			errMsg := fmt.Sprintf("got status %q from blobserver URL %q during configuration discovery", res.Status, c.discoRoot())
  1057			if res.StatusCode == 401 && c.authErr != nil {
  1058				errMsg = fmt.Sprintf("%v. %v", c.authErr, errMsg)
  1059			}
  1060			return nil, errors.New(errMsg)
  1061		}
  1062		// TODO(bradfitz): little weird in retrospect that we request
  1063		// text/x-camli-configuration and expect to get back
  1064		// text/javascript.  Make them consistent.
  1065		if ct := res.Header.Get("Content-Type"); ct != "text/javascript" {
  1066			res.Body.Close()
  1067			return nil, fmt.Errorf("Blobserver returned unexpected type %q from discovery", ct)
  1068		}
  1069		return res, nil
  1070	}
  1071	
  1072	func (c *Client) doDiscovery() error {
  1073		ctx := context.TODO()
  1074		root, err := url.Parse(c.discoRoot())
  1075		if err != nil {
  1076			return err
  1077		}
  1078	
  1079		res, err := c.discoveryResp(ctx)
  1080		if err != nil {
  1081			return err
  1082		}
  1083	
  1084		var disco camtypes.Discovery
  1085		if err := httputil.DecodeJSON(res, &disco); err != nil {
  1086			return err
  1087		}
  1088	
  1089		if disco.SearchRoot == "" {
  1090			c.searchRoot = ""
  1091		} else {
  1092			u, err := root.Parse(disco.SearchRoot)
  1093			if err != nil {
  1094				return fmt.Errorf("client: invalid searchRoot %q; failed to resolve", disco.SearchRoot)
  1095			}
  1096			c.searchRoot = u.String()
  1097		}
  1098	
  1099		u, err := root.Parse(disco.HelpRoot)
  1100		if err != nil {
  1101			return fmt.Errorf("client: invalid helpRoot %q; failed to resolve", disco.HelpRoot)
  1102		}
  1103		c.helpRoot = u.String()
  1104	
  1105		u, err = root.Parse(disco.ShareRoot)
  1106		if err != nil {
  1107			return fmt.Errorf("client: invalid shareRoot %q; failed to resolve", disco.ShareRoot)
  1108		}
  1109		c.shareRoot = u.String()
  1110	
  1111		c.storageGen = disco.StorageGeneration
  1112		c.hasLegacySHA1 = disco.HasLegacySHA1Index
  1113	
  1114		u, err = root.Parse(disco.BlobRoot)
  1115		if err != nil {
  1116			return fmt.Errorf("client: error resolving blobRoot: %v", err)
  1117		}
  1118		c.prefixv = strings.TrimRight(u.String(), "/")
  1119	
  1120		if disco.UIDiscovery != nil {
  1121			u, err = root.Parse(disco.DownloadHelper)
  1122			if err != nil {
  1123				return fmt.Errorf("client: invalid downloadHelper %q; failed to resolve", disco.DownloadHelper)
  1124			}
  1125			c.downloadHelper = u.String()
  1126		}
  1127	
  1128		if disco.SyncHandlers != nil {
  1129			for _, v := range disco.SyncHandlers {
  1130				ufrom, err := root.Parse(v.From)
  1131				if err != nil {
  1132					return fmt.Errorf("client: invalid %q \"from\" sync; failed to resolve", v.From)
  1133				}
  1134				uto, err := root.Parse(v.To)
  1135				if err != nil {
  1136					return fmt.Errorf("client: invalid %q \"to\" sync; failed to resolve", v.To)
  1137				}
  1138				c.syncHandlers = append(c.syncHandlers, &SyncInfo{
  1139					From:    ufrom.String(),
  1140					To:      uto.String(),
  1141					ToIndex: v.ToIndex,
  1142				})
  1143			}
  1144		}
  1145	
  1146		if disco.Signing != nil {
  1147			c.serverKeyID = disco.Signing.PublicKeyID
  1148			c.serverPublicKeyBlobRef = disco.Signing.PublicKeyBlobRef
  1149			c.signHandler = disco.Signing.SignHandler
  1150		}
  1151		return nil
  1152	}
  1153	
  1154	// GetJSON sends a GET request to url, and unmarshals the returned
  1155	// JSON response into data. The URL's host must match the client's
  1156	// configured server.
  1157	func (c *Client) GetJSON(ctx context.Context, url string, data interface{}) error {
  1158		if !strings.HasPrefix(url, c.discoRoot()) {
  1159			return fmt.Errorf("wrong URL (%q) for this server", url)
  1160		}
  1161		hreq := c.newRequest(ctx, "GET", url)
  1162		resp, err := c.expect2XX(hreq)
  1163		if err != nil {
  1164			return err
  1165		}
  1166		return httputil.DecodeJSON(resp, data)
  1167	}
  1168	
  1169	// Post is like http://golang.org/pkg/net/http/#Client.Post
  1170	// but with implementation details like gated requests. The
  1171	// URL's host must match the client's configured server.
  1172	func (c *Client) Post(ctx context.Context, url string, bodyType string, body io.Reader) error {
  1173		resp, err := c.post(ctx, url, bodyType, body)
  1174		if err != nil {
  1175			return err
  1176		}
  1177		return resp.Body.Close()
  1178	}
  1179	
  1180	// Sign sends a request to the sign handler on server to sign the contents of r,
  1181	// and return them signed. It uses the same implementation details, such as gated
  1182	// requests, as Post.
  1183	func (c *Client) Sign(ctx context.Context, server string, r io.Reader) (signed []byte, err error) {
  1184		signHandler, err := c.SignHandler()
  1185		if err != nil {
  1186			return nil, err
  1187		}
  1188		signServer := strings.TrimSuffix(server, "/") + signHandler
  1189		resp, err := c.post(ctx, signServer, "application/x-www-form-urlencoded", r)
  1190		if err != nil {
  1191			return nil, err
  1192		}
  1193		defer resp.Body.Close()
  1194		return ioutil.ReadAll(resp.Body)
  1195	}
  1196	
  1197	func (c *Client) post(ctx context.Context, url string, bodyType string, body io.Reader) (*http.Response, error) {
  1198		if !c.sameOrigin && !strings.HasPrefix(url, c.discoRoot()) {
  1199			return nil, fmt.Errorf("wrong URL (%q) for this server", url)
  1200		}
  1201		req := c.newRequest(ctx, "POST", url, body)
  1202		req.Header.Set("Content-Type", bodyType)
  1203		res, err := c.expect2XX(req)
  1204		if err != nil {
  1205			return nil, err
  1206		}
  1207		return res, nil
  1208	}
  1209	
  1210	// newRequest creates a request with the authentication header, and with the
  1211	// appropriate scheme and port in the case of self-signed TLS.
  1212	func (c *Client) newRequest(ctx context.Context, method, url string, body ...io.Reader) *http.Request {
  1213		var bodyR io.Reader
  1214		if len(body) > 0 {
  1215			bodyR = body[0]
  1216		}
  1217		if len(body) > 1 {
  1218			panic("too many body arguments")
  1219		}
  1220		req, err := http.NewRequest(method, url, bodyR)
  1221		if err != nil {
  1222			panic(err.Error())
  1223		}
  1224		// not done by http.NewRequest in Go 1.0:
  1225		if br, ok := bodyR.(*bytes.Reader); ok {
  1226			req.ContentLength = int64(br.Len())
  1227		}
  1228		c.authMode.AddAuthHeader(req)
  1229		return req.WithContext(ctx)
  1230	}
  1231	
  1232	// expect2XX will doReqGated and promote HTTP response codes outside of
  1233	// the 200-299 range to a non-nil error containing the response body.
  1234	func (c *Client) expect2XX(req *http.Request) (*http.Response, error) {
  1235		res, err := c.doReqGated(req)
  1236		if err == nil && (res.StatusCode < 200 || res.StatusCode > 299) {
  1237			buf := new(bytes.Buffer)
  1238			io.CopyN(buf, res.Body, 1<<20)
  1239			res.Body.Close()
  1240			return res, fmt.Errorf("client: got status code %d from URL %s; body %s", res.StatusCode, req.URL.String(), buf.String())
  1241		}
  1242		return res, err
  1243	}
  1244	
  1245	func (c *Client) doReqGated(req *http.Request) (*http.Response, error) {
  1246		c.httpGate.Start()
  1247		defer c.httpGate.Done()
  1248		return c.httpClient.Do(req)
  1249	}
  1250	
  1251	// DialFunc returns the adequate dial function when we're on android.
  1252	func (c *Client) DialFunc() func(network, addr string) (net.Conn, error) {
  1253		if c.useTLS() {
  1254			return nil
  1255		}
  1256		if android.IsChild() {
  1257			return func(network, addr string) (net.Conn, error) {
  1258				return android.Dial(network, addr)
  1259			}
  1260		}
  1261		return nil
  1262	}
  1263	
  1264	func (c *Client) http2DialTLSFunc() func(network, addr string, cfg *tls.Config) (net.Conn, error) {
  1265		trustedCerts := c.getTrustedCerts()
  1266		if !c.insecureAnyTLSCert && len(trustedCerts) == 0 {
  1267			// TLS with normal/full verification.
  1268			// nil means http2 uses its default dialer.
  1269			return nil
  1270		}
  1271		return func(network, addr string, cfg *tls.Config) (net.Conn, error) {
  1272			// we own cfg, so we can mutate it:
  1273			cfg.InsecureSkipVerify = true
  1274			conn, err := tls.Dial(network, addr, cfg)
  1275			if err != nil {
  1276				return nil, err
  1277			}
  1278			if c.insecureAnyTLSCert {
  1279				return conn, err
  1280			}
  1281			state := conn.ConnectionState()
  1282			if p := state.NegotiatedProtocol; p != http2.NextProtoTLS {
  1283				return nil, fmt.Errorf("http2: unexpected ALPN protocol %q; want %q", p, http2.NextProtoTLS)
  1284			}
  1285			if !state.NegotiatedProtocolIsMutual {
  1286				return nil, errors.New("http2: could not negotiate protocol mutually")
  1287			}
  1288			certs := state.PeerCertificates
  1289			if len(certs) < 1 {
  1290				return nil, fmt.Errorf("no TLS peer certificates from %s", addr)
  1291			}
  1292			sig := hashutil.SHA256Prefix(certs[0].Raw)
  1293			for _, v := range trustedCerts {
  1294				if v == sig {
  1295					return conn, nil
  1296				}
  1297			}
  1298			return nil, fmt.Errorf("TLS server at %v presented untrusted certificate (signature %q)", addr, sig)
  1299		}
  1300	}
  1301	
  1302	// DialTLSFunc returns the adequate dial function, when using SSL, depending on
  1303	// whether we're using insecure TLS (certificate verification is disabled), or we
  1304	// have some trusted certs, or we're on android.
  1305	// If the client's config has some trusted certs, the server's certificate will
  1306	// be checked against those in the config after the TLS handshake.
  1307	func (c *Client) DialTLSFunc() func(network, addr string) (net.Conn, error) {
  1308		if !c.useTLS() {
  1309			return nil
  1310		}
  1311		trustedCerts := c.getTrustedCerts()
  1312		var stdTLS bool
  1313		if !c.insecureAnyTLSCert && len(trustedCerts) == 0 {
  1314			// TLS with normal/full verification.
  1315			stdTLS = true
  1316			if !android.IsChild() {
  1317				// Not android, so let the stdlib deal with it
  1318				return nil
  1319			}
  1320		}
  1321	
  1322		return func(network, addr string) (net.Conn, error) {
  1323			var conn *tls.Conn
  1324			var err error
  1325			if android.IsChild() {
  1326				ac, err := android.Dial(network, addr)
  1327				if err != nil {
  1328					return nil, err
  1329				}
  1330				var tlsConfig *tls.Config
  1331				if stdTLS {
  1332					tlsConfig, err = android.TLSConfig()
  1333					if err != nil {
  1334						return nil, err
  1335					}
  1336				} else {
  1337					tlsConfig = &tls.Config{InsecureSkipVerify: true}
  1338				}
  1339				// Since we're doing the TLS handshake ourselves, we need to set the ServerName,
  1340				// in case the server uses SNI (as is the case if it's relying on Let's Encrypt,
  1341				// for example).
  1342				tlsConfig.ServerName = c.serverNameOfAddr(addr)
  1343				conn = tls.Client(ac, tlsConfig)
  1344				if err := conn.Handshake(); err != nil {
  1345					return nil, err
  1346				}
  1347				if stdTLS {
  1348					// Normal TLS verification succeeded and we do not have
  1349					// additional trusted certificate fingerprints to check for.
  1350					return conn, nil
  1351				}
  1352			} else {
  1353				conn, err = tls.Dial(network, addr, &tls.Config{InsecureSkipVerify: true})
  1354				if err != nil {
  1355					return nil, err
  1356				}
  1357			}
  1358			if c.insecureAnyTLSCert {
  1359				return conn, nil
  1360			}
  1361			certs := conn.ConnectionState().PeerCertificates
  1362			if len(certs) < 1 {
  1363				return nil, fmt.Errorf("no TLS peer certificates from %s", addr)
  1364			}
  1365			sig := hashutil.SHA256Prefix(certs[0].Raw)
  1366			for _, v := range trustedCerts {
  1367				if v == sig {
  1368					return conn, nil
  1369				}
  1370			}
  1371			return nil, fmt.Errorf("TLS server at %v presented untrusted certificate (signature %q)", addr, sig)
  1372		}
  1373	}
  1374	
  1375	// serverNameOfAddr returns the host part of addr, or the empty string if addr
  1376	// is not a valid address (see net.Dial). Additionally, if host is an IP literal,
  1377	// serverNameOfAddr returns the empty string.
  1378	func (c *Client) serverNameOfAddr(addr string) string {
  1379		serverName, _, err := net.SplitHostPort(addr)
  1380		if err != nil {
  1381			c.printf("could not get server name from address %q: %v", addr, err)
  1382			return ""
  1383		}
  1384		if ip := net.ParseIP(serverName); ip != nil {
  1385			return ""
  1386		}
  1387		return serverName
  1388	}
  1389	
  1390	// Signer returns the client's Signer, if any. The Signer signs JSON
  1391	// mutation claims.
  1392	func (c *Client) Signer() (*schema.Signer, error) {
  1393		c.signerOnce.Do(c.signerInit)
  1394		return c.signer, c.signerErr
  1395	}
  1396	
  1397	func (c *Client) signerInit() {
  1398		c.signer, c.signerErr = c.buildSigner()
  1399	}
  1400	
  1401	func (c *Client) buildSigner() (*schema.Signer, error) {
  1402		c.initSignerPublicKeyBlobrefOnce.Do(c.initSignerPublicKeyBlobref)
  1403		if !c.signerPublicKeyRef.Valid() {
  1404			return nil, camtypes.ErrClientNoPublicKey
  1405		}
  1406		return schema.NewSigner(c.signerPublicKeyRef, strings.NewReader(c.publicKeyArmored), c.SecretRingFile())
  1407	}
  1408	
  1409	// sigTime optionally specifies the signature time.
  1410	// If zero, the current time is used.
  1411	func (c *Client) signBlob(ctx context.Context, bb schema.Buildable, sigTime time.Time) (string, error) {
  1412		signer, err := c.Signer()
  1413		if err != nil {
  1414			return "", err
  1415		}
  1416		return bb.Builder().SignAt(ctx, signer, sigTime)
  1417	}
  1418	
  1419	// uploadPublicKey uploads the public key (if one is defined), so
  1420	// subsequent (likely synchronous) indexing of uploaded signed blobs
  1421	// will have access to the public key to verify it. In the normal
  1422	// case, the stat cache prevents this from doing anything anyway.
  1423	func (c *Client) uploadPublicKey(ctx context.Context) error {
  1424		sigRef := c.SignerPublicKeyBlobref()
  1425		if !sigRef.Valid() {
  1426			return nil
  1427		}
  1428		var err error
  1429		if _, keyUploaded := c.haveCache.StatBlobCache(sigRef); !keyUploaded {
  1430			_, err = c.uploadString(ctx, c.publicKeyArmored, false)
  1431		}
  1432		return err
  1433	}
  1434	
  1435	// checkMatchingKeys compares the client's and the server's keys and logs if they differ.
  1436	func (c *Client) checkMatchingKeys() {
  1437		serverKey, err := c.ServerKeyID()
  1438		if err != nil {
  1439			log.Printf("Warning: Could not obtain ther server's key id: %v", err)
  1440			return
  1441		}
  1442		if serverKey != c.signer.KeyIDLong() {
  1443			log.Printf("Warning: client (%s) and server (%s) keys differ.", c.signer.KeyIDLong(), serverKey)
  1444		}
  1445	}
  1446	
  1447	func (c *Client) UploadAndSignBlob(ctx context.Context, b schema.AnyBlob) (*PutResult, error) {
  1448		signed, err := c.signBlob(ctx, b.Blob(), time.Time{})
  1449		if err != nil {
  1450			return nil, err
  1451		}
  1452		c.checkMatchingKeys()
  1453		if err := c.uploadPublicKey(ctx); err != nil {
  1454			return nil, err
  1455		}
  1456		return c.uploadString(ctx, signed, false)
  1457	}
  1458	
  1459	func (c *Client) UploadBlob(ctx context.Context, b schema.AnyBlob) (*PutResult, error) {
  1460		// TODO(bradfitz): ask the blob for its own blobref, rather
  1461		// than changing the hash function with uploadString?
  1462		return c.uploadString(ctx, b.Blob().JSON(), true)
  1463	}
  1464	
  1465	func (c *Client) uploadString(ctx context.Context, s string, stat bool) (*PutResult, error) {
  1466		uh := NewUploadHandleFromString(s)
  1467		uh.SkipStat = !stat
  1468		return c.Upload(ctx, uh)
  1469	}
  1470	
  1471	func (c *Client) UploadNewPermanode(ctx context.Context) (*PutResult, error) {
  1472		unsigned := schema.NewUnsignedPermanode()
  1473		return c.UploadAndSignBlob(ctx, unsigned)
  1474	}
  1475	
  1476	func (c *Client) UploadPlannedPermanode(ctx context.Context, key string, sigTime time.Time) (*PutResult, error) {
  1477		unsigned := schema.NewPlannedPermanode(key)
  1478		signed, err := c.signBlob(ctx, unsigned, sigTime)
  1479		if err != nil {
  1480			return nil, err
  1481		}
  1482		c.checkMatchingKeys()
  1483		if err := c.uploadPublicKey(ctx); err != nil {
  1484			return nil, err
  1485		}
  1486		return c.uploadString(ctx, signed, true)
  1487	}
  1488	
  1489	// IsIgnoredFile returns whether the file at fullpath should be ignored by pk-put.
  1490	// The fullpath is checked against the ignoredFiles list, trying the following rules in this order:
  1491	// 1) star-suffix style matching (.e.g *.jpg).
  1492	// 2) Shell pattern match as done by http://golang.org/pkg/path/filepath/#Match
  1493	// 3) If the pattern is an absolute path to a directory, fullpath matches if it is that directory or a child of it.
  1494	// 4) If the pattern is a relative path, fullpath matches if it has pattern as a path component (i.e the pattern is a part of fullpath that fits exactly between two path separators).
  1495	func (c *Client) IsIgnoredFile(fullpath string) bool {
  1496		c.initIgnoredFilesOnce.Do(c.initIgnoredFiles)
  1497		return c.ignoreChecker(fullpath)
  1498	}
  1499	
  1500	// Close closes the client.
  1501	func (c *Client) Close() error {
  1502		if cl, ok := c.sto.(io.Closer); ok {
  1503			return cl.Close()
  1504		}
  1505		if c := c.HTTPClient(); c != nil {
  1506			switch t := c.Transport.(type) {
  1507			case *http.Transport:
  1508				t.CloseIdleConnections()
  1509			case *http2.Transport:
  1510				t.CloseIdleConnections()
  1511			}
  1512		}
  1513		return nil
  1514	}
  1515	
  1516	func httpGateSize(rt http.RoundTripper) int {
  1517		switch v := rt.(type) {
  1518		case *httputil.StatsTransport:
  1519			return httpGateSize(v.Transport)
  1520		case *http.Transport:
  1521			return maxParallelHTTP_h1
  1522		case *http2.Transport:
  1523			return maxParallelHTTP_h2
  1524		default:
  1525			return maxParallelHTTP_h1 // conservative default
  1526		}
  1527	}
Website layout inspired by memcached.
Content by the authors.