1	/*
     2	Copyright 2011 The Perkeep Authors.
     3	
     4	Licensed under the Apache License, Version 2.0 (the "License");
     5	you may not use this file except in compliance with the License.
     6	You may obtain a copy of the License at
     7	
     8	     http://www.apache.org/licenses/LICENSE-2.0
     9	
    10	Unless required by applicable law or agreed to in writing, software
    11	distributed under the License is distributed on an "AS IS" BASIS,
    12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13	See the License for the specific language governing permissions and
    14	limitations under the License.
    15	*/
    16	
    17	// Package client implements a Perkeep client.
    18	package client // import "perkeep.org/pkg/client"
    19	
    20	import (
    21		"bytes"
    22		"context"
    23		"crypto/tls"
    24		"encoding/json"
    25		"errors"
    26		"fmt"
    27		"io"
    28		"log"
    29		"net"
    30		"net/http"
    31		"net/url"
    32		"os"
    33		"regexp"
    34		"slices"
    35		"strings"
    36		"sync"
    37		"time"
    38	
    39		"perkeep.org/internal/hashutil"
    40		"perkeep.org/internal/httputil"
    41		"perkeep.org/internal/osutil"
    42		"perkeep.org/pkg/auth"
    43		"perkeep.org/pkg/blob"
    44		"perkeep.org/pkg/blobserver"
    45		"perkeep.org/pkg/buildinfo"
    46		"perkeep.org/pkg/client/android"
    47		"perkeep.org/pkg/schema"
    48		"perkeep.org/pkg/search"
    49		"perkeep.org/pkg/types/camtypes"
    50	
    51		"go4.org/syncutil"
    52		"golang.org/x/net/http2"
    53	)
    54	
    55	// A Client provides access to a Perkeep server.
    56	//
    57	// After use, a Client should be closed via its Close method to
    58	// release idle HTTP connections or other resourcedds.
    59	type Client struct {
    60		// server is the input from user, pre-discovery.
    61		// For example "http://foo.com" or "foo.com:1234".
    62		// It is the responsibility of initPrefix to parse
    63		// server and set prefix, including doing discovery
    64		// to figure out what the proper server-declared
    65		// prefix is.
    66		server string
    67	
    68		prefixOnce    syncutil.Once // guards init of following 2 fields
    69		prefixv       string        // URL prefix before "/camli/"
    70		isSharePrefix bool          // URL is a request for a share blob
    71	
    72		discoOnce              syncutil.Once
    73		searchRoot             string      // Handler prefix, or "" if none
    74		downloadHelper         string      // or "" if none
    75		storageGen             string      // storage generation, or "" if not reported
    76		hasLegacySHA1          bool        // Whether server has SHA-1 blobs indexed.
    77		syncHandlers           []*SyncInfo // "from" and "to" url prefix for each syncHandler
    78		serverKeyID            string      // Server's GPG public key ID.
    79		helpRoot               string      // Handler prefix, or "" if none
    80		shareRoot              string      // Share handler prefix, or "" if none
    81		serverPublicKeyBlobRef blob.Ref    // Server's public key blobRef
    82	
    83		signerOnce  sync.Once
    84		signer      *schema.Signer
    85		signerErr   error
    86		signHandler string // Handler prefix, or "" if none
    87	
    88		authMode auth.AuthMode
    89		// authErr is set when no auth config is found but we want to defer warning
    90		// until discovery fails.
    91		authErr error
    92	
    93		httpClient *http.Client
    94		haveCache  HaveCache
    95	
    96		// If sto is set, it's used before the httpClient or other network operations.
    97		sto blobserver.Storage
    98	
    99		initTrustedCertsOnce sync.Once
   100	
   101		// We define a certificate fingerprint as the 20 digits lowercase prefix
   102		// of the SHA256 of the complete certificate (in ASN.1 DER encoding).
   103		// trustedCerts contains the fingerprints of the self-signed
   104		// certificates we trust.
   105		// If not empty, (and if using TLS) the full x509 verification is
   106		// disabled, and we instead check the server's certificate against
   107		// this list.
   108		// The perkeepd server prints the fingerprint to add to the config
   109		// when starting.
   110		trustedCerts []string
   111	
   112		// insecureAnyTLSCert disables all TLS cert checking,
   113		// including the trustedCerts field above.
   114		insecureAnyTLSCert bool
   115	
   116		initIgnoredFilesOnce sync.Once
   117		// list of files that pk-put should ignore.
   118		// Defaults to empty, but pk-put init creates a config with a non
   119		// empty list.
   120		// See IsIgnoredFile for the matching rules.
   121		ignoredFiles  []string
   122		ignoreChecker func(path string) bool
   123	
   124		initSignerPublicKeyBlobrefOnce sync.Once
   125		signerPublicKeyRef             blob.Ref
   126		publicKeyArmored               string
   127	
   128		statsMutex sync.Mutex
   129		stats      Stats
   130	
   131		// via maps the access path from a share root to a desired target.
   132		// It is non-nil when in "sharing" mode, where the Client is fetching
   133		// a share.
   134		viaMu sync.RWMutex
   135		via   map[blob.Ref]blob.Ref // target => via (target is referenced from via)
   136	
   137		// Verbose controls how much logging from the client is printed. The caller
   138		// should set it only before using the client, and treat it as read-only after
   139		// that.
   140		Verbose bool
   141		// Logger is the logger used by the client. It defaults to a standard
   142		// logger to os.Stderr if the client is initialized by one of the package's
   143		// functions. Like Verbose, it should be set only before using the client, and
   144		// not be modified afterwards.
   145		Logger *log.Logger
   146	
   147		httpGate        *syncutil.Gate
   148		transportConfig *TransportConfig // or nil
   149	
   150		noExtConfig bool // no external config; config file and env vars are ignored.
   151	
   152		// sameOrigin indicates whether URLs in requests should be stripped from
   153		// their scheme and HostPort parts. This is meant for when using the client
   154		// through gopherjs in the web UI. Because we'll run into CORS errors if
   155		// requests have a Host part.
   156		sameOrigin bool
   157	}
   158	
   159	const maxParallelHTTP_h1 = 5
   160	const maxParallelHTTP_h2 = 50
   161	
   162	// New returns a new Perkeep Client.
   163	//
   164	// By default, with no options, it uses the client as configured in
   165	// the environment or default configuration files.
   166	func New(opts ...ClientOption) (*Client, error) {
   167		c := &Client{
   168			haveCache: noHaveCache{},
   169			Logger:    log.New(os.Stderr, "", log.Ldate|log.Ltime),
   170			authMode:  auth.None{},
   171		}
   172		for _, v := range opts {
   173			v.modifyClient(c)
   174		}
   175		if c.sto != nil && len(opts) > 1 {
   176			return nil, errors.New("use of OptionUseStorageClient precludes use of any other options")
   177		}
   178	
   179		if c.noExtConfig {
   180			c.setDefaultHTTPClient()
   181			return c, nil
   182		}
   183	
   184		if c.server != "" {
   185			if !isURLOrHostPort(c.server) {
   186				configOnce.Do(parseConfig)
   187				serverConf, ok := config.Servers[c.server]
   188				if !ok {
   189					log.Fatalf("%q looks like a server alias, but no such alias found in config at %v", c.server, osutil.UserClientConfigPath())
   190				}
   191				c.server = serverConf.Server
   192			}
   193			c.setDefaultHTTPClient()
   194			return c, nil
   195		}
   196	
   197		var err error
   198		c.server, err = getServer()
   199		if err != nil {
   200			return nil, err
   201		}
   202		err = c.SetupAuth()
   203		if err != nil {
   204			return nil, err
   205		}
   206		c.setDefaultHTTPClient()
   207		return c, nil
   208	}
   209	
   210	func (c *Client) setDefaultHTTPClient() {
   211		if c.httpClient == nil {
   212			c.httpClient = &http.Client{
   213				Transport: c.transportForConfig(c.transportConfig),
   214			}
   215		}
   216		c.httpGate = syncutil.NewGate(httpGateSize(c.httpClient.Transport))
   217	}
   218	
   219	// NewOrFail is like New, but calls log.Fatal instead of returning an error.
   220	func NewOrFail(opts ...ClientOption) *Client {
   221		c, err := New(opts...)
   222		if err != nil {
   223			log.Fatalf("error creating client: %v", err)
   224		}
   225		return c
   226	}
   227	
   228	// NewPathClient returns a new client accessing a subpath of c.
   229	func (c *Client) NewPathClient(path string) (*Client, error) {
   230		u, err := url.Parse(c.server)
   231		if err != nil {
   232			return nil, fmt.Errorf("bogus server %q for NewPathClient receiver: %w", c.server, err)
   233		}
   234		u.Path = path
   235		pc, err := New(OptionServer(u.String()))
   236		if err != nil {
   237			return nil, err
   238		}
   239		pc.authMode = c.authMode
   240		pc.discoOnce.Do(noop)
   241		return pc, nil
   242	}
   243	
   244	// TransportConfig contains options for how HTTP requests are made.
   245	type TransportConfig struct {
   246		// Proxy optionally specifies the Proxy for the transport. Useful with
   247		// pk-put for debugging even localhost requests.
   248		Proxy   func(*http.Request) (*url.URL, error)
   249		Verbose bool // Verbose enables verbose logging of HTTP requests.
   250	}
   251	
   252	func (c *Client) useHTTP2(tc *TransportConfig) bool {
   253		if !c.useTLS() {
   254			return false
   255		}
   256		if android.IsChild() {
   257			// No particular reason; just untested so far.
   258			return false
   259		}
   260		if os.Getenv("HTTPS_PROXY") != "" || os.Getenv("https_proxy") != "" ||
   261			(tc != nil && tc.Proxy != nil) {
   262			// Also just untested. Which proxies support h2 anyway?
   263			return false
   264		}
   265		return true
   266	}
   267	
   268	// transportForConfig returns a transport for the client, setting the correct
   269	// Proxy, Dial, and TLSClientConfig if needed. It does not mutate c.
   270	// It is the caller's responsibility to then use that transport to set
   271	// the client's httpClient with SetHTTPClient.
   272	func (c *Client) transportForConfig(tc *TransportConfig) http.RoundTripper {
   273		if c == nil {
   274			return nil
   275		}
   276		var transport http.RoundTripper
   277		proxy := http.ProxyFromEnvironment
   278		if tc != nil && tc.Proxy != nil {
   279			proxy = tc.Proxy
   280		}
   281	
   282		if c.useHTTP2(tc) {
   283			transport = &http2.Transport{
   284				DialTLS: c.http2DialTLSFunc(),
   285			}
   286		} else {
   287			transport = &http.Transport{
   288				DialTLS:             c.DialTLSFunc(),
   289				Dial:                c.DialFunc(),
   290				Proxy:               proxy,
   291				MaxIdleConnsPerHost: maxParallelHTTP_h1,
   292			}
   293		}
   294		httpStats := &httputil.StatsTransport{
   295			Transport: transport,
   296		}
   297		if tc != nil {
   298			httpStats.VerboseLog = tc.Verbose
   299		}
   300		transport = httpStats
   301		if android.IsChild() {
   302			transport = &android.StatsTransport{Rt: transport}
   303		}
   304		return transport
   305	}
   306	
   307	// HTTPStats returns the client's underlying httputil.StatsTransport, if in use.
   308	// If another transport is being used, nil is returned.
   309	func (c *Client) HTTPStats() *httputil.StatsTransport {
   310		st, _ := c.httpClient.Transport.(*httputil.StatsTransport)
   311		return st
   312	}
   313	
   314	type ClientOption interface {
   315		modifyClient(*Client)
   316	}
   317	
   318	// OptionServer returns a Client constructor option that forces use of
   319	// the provided server.
   320	//
   321	// The provided server is either "host:port" (assumed http, not https)
   322	// or a URL prefix, with or without a path, or a server alias from the
   323	// client configuration file. A server alias should not be confused
   324	// with a hostname, therefore it cannot contain any colon or period.
   325	func OptionServer(server string) ClientOption {
   326		return optionServer(server)
   327	}
   328	
   329	type optionServer string
   330	
   331	func (s optionServer) modifyClient(c *Client) { c.server = string(s) }
   332	
   333	// OptionUseStorageClient returns a Client constructor option that
   334	// forces use of the provided blob storage target.
   335	//
   336	// This exists mainly so all the convenience methods on
   337	// Client (e.g. the Upload variants) are available against storage
   338	// directly.
   339	//
   340	// Use of OptionUseStorageClient is mutually exclusively
   341	// with all other options, although it does imply
   342	// OptionNoExternalConfig(true).
   343	func OptionUseStorageClient(s blobserver.Storage) ClientOption {
   344		return optionStorage{s}
   345	}
   346	
   347	type optionStorage struct {
   348		sto blobserver.Storage
   349	}
   350	
   351	func (o optionStorage) modifyClient(c *Client) {
   352		c.sto = o.sto
   353		c.noExtConfig = true
   354	}
   355	
   356	// OptionTransportConfig returns a ClientOption that makes the client use
   357	// the provided transport configuration options.
   358	func OptionTransportConfig(tc *TransportConfig) ClientOption {
   359		return optionTransportConfig{tc}
   360	}
   361	
   362	type optionTransportConfig struct {
   363		tc *TransportConfig
   364	}
   365	
   366	func (o optionTransportConfig) modifyClient(c *Client) { c.transportConfig = o.tc }
   367	
   368	// OptionInsecure returns a ClientOption that controls whether HTTP
   369	// requests are allowed to be insecure (over HTTP or HTTPS without TLS
   370	// certificate checking). Use of this is strongly discouraged except
   371	// for local testing.
   372	func OptionInsecure(v bool) ClientOption {
   373		return optionInsecure(v)
   374	}
   375	
   376	type optionInsecure bool
   377	
   378	func (o optionInsecure) modifyClient(c *Client) { c.insecureAnyTLSCert = bool(o) }
   379	
   380	// OptionTrustedCert returns a ClientOption that makes the client
   381	// trust the provide self-signed cert signature. The value should be
   382	// the 20 byte hex prefix of the SHA-256 of the cert, as printed by
   383	// the perkeepd server on start-up.
   384	//
   385	// If cert is empty, the option has no effect.
   386	func OptionTrustedCert(cert string) ClientOption {
   387		// TODO: remove this whole function now that we have LetsEncrypt?
   388		return optionTrustedCert(cert)
   389	}
   390	
   391	type optionTrustedCert string
   392	
   393	func (o optionTrustedCert) modifyClient(c *Client) {
   394		cert := string(o)
   395		if cert != "" {
   396			c.initTrustedCertsOnce.Do(func() {})
   397			c.trustedCerts = []string{string(o)}
   398		}
   399	}
   400	
   401	type optionNoExtConfig bool
   402	
   403	func (o optionNoExtConfig) modifyClient(c *Client) { c.noExtConfig = bool(o) }
   404	
   405	// OptionNoExternalConfig returns a Client constructor option that
   406	// prevents any on-disk config files or environment variables from
   407	// influencing the client. It may still use the disk for caches.
   408	func OptionNoExternalConfig() ClientOption {
   409		return optionNoExtConfig(true)
   410	}
   411	
   412	type optionAuthMode struct {
   413		m auth.AuthMode
   414	}
   415	
   416	func (o optionAuthMode) modifyClient(c *Client) { c.authMode = o.m }
   417	
   418	// OptionAuthMode returns a Client constructor option that sets the
   419	// client's authentication mode.
   420	func OptionAuthMode(m auth.AuthMode) ClientOption {
   421		return optionAuthMode{m}
   422	}
   423	
   424	// noop is for use with syncutil.Onces.
   425	func noop() error { return nil }
   426	
   427	var shareURLRx = regexp.MustCompile(`^(.+)/(` + blob.Pattern + ")$")
   428	
   429	// NewFromShareRoot uses shareBlobURL to set up and return a client that
   430	// will be used to fetch shared blobs.
   431	func NewFromShareRoot(ctx context.Context, shareBlobURL string, opts ...ClientOption) (c *Client, target blob.Ref, err error) {
   432		var root string
   433		m := shareURLRx.FindStringSubmatch(shareBlobURL)
   434		if m == nil {
   435			return nil, blob.Ref{}, fmt.Errorf("Unknown share URL base")
   436		}
   437		c, err = New(append(opts[:len(opts):cap(opts)], OptionServer(m[1]))...)
   438		if err != nil {
   439			return nil, blob.Ref{}, err
   440		}
   441		c.discoOnce.Do(noop)
   442		c.prefixOnce.Do(noop)
   443		c.prefixv = m[1]
   444		c.isSharePrefix = true
   445		c.authMode = auth.None{}
   446		c.via = make(map[blob.Ref]blob.Ref)
   447		root = m[2]
   448	
   449		req := c.newRequest(ctx, "GET", shareBlobURL, nil)
   450		res, err := c.expect2XX(req)
   451		if err != nil {
   452			return nil, blob.Ref{}, fmt.Errorf("error fetching %s: %w", shareBlobURL, err)
   453		}
   454		defer res.Body.Close()
   455		var buf bytes.Buffer
   456		rootbr, ok := blob.Parse(root)
   457		if !ok {
   458			return nil, blob.Ref{}, fmt.Errorf("invalid root blob ref for sharing: %q", root)
   459		}
   460		b, err := schema.BlobFromReader(rootbr, io.TeeReader(res.Body, &buf))
   461		if err != nil {
   462			return nil, blob.Ref{}, fmt.Errorf("error parsing JSON from %s: %w, with response: %q", shareBlobURL, err, buf.Bytes())
   463		}
   464		if b.ShareAuthType() != schema.ShareHaveRef {
   465			return nil, blob.Ref{}, fmt.Errorf("unknown share authType of %q", b.ShareAuthType())
   466		}
   467		target = b.ShareTarget()
   468		if !target.Valid() {
   469			return nil, blob.Ref{}, fmt.Errorf("no target")
   470		}
   471		c.via[target] = rootbr
   472		return c, target, nil
   473	}
   474	
   475	// SetHTTPClient sets the Perkeep client's HTTP client.
   476	// If nil, the default HTTP client is used.
   477	func (c *Client) SetHTTPClient(client *http.Client) {
   478		if client == nil {
   479			client = http.DefaultClient
   480		}
   481		c.httpClient = client
   482	}
   483	
   484	// HTTPClient returns the Client's underlying http.Client.
   485	func (c *Client) HTTPClient() *http.Client {
   486		return c.httpClient
   487	}
   488	
   489	// A HaveCache caches whether a remote blobserver has a blob.
   490	type HaveCache interface {
   491		StatBlobCache(br blob.Ref) (size uint32, ok bool)
   492		NoteBlobExists(br blob.Ref, size uint32)
   493	}
   494	
   495	type noHaveCache struct{}
   496	
   497	func (noHaveCache) StatBlobCache(blob.Ref) (uint32, bool) { return 0, false }
   498	func (noHaveCache) NoteBlobExists(blob.Ref, uint32)       {}
   499	
   500	func (c *Client) SetHaveCache(cache HaveCache) {
   501		if cache == nil {
   502			cache = noHaveCache{}
   503		}
   504		c.haveCache = cache
   505	}
   506	
   507	func (c *Client) printf(format string, v ...interface{}) {
   508		if c.Verbose && c.Logger != nil {
   509			c.Logger.Printf(format, v...)
   510		}
   511	}
   512	
   513	func (c *Client) Stats() Stats {
   514		c.statsMutex.Lock()
   515		defer c.statsMutex.Unlock()
   516		return c.stats // copy
   517	}
   518	
   519	// ErrNoSearchRoot is returned by SearchRoot if the server doesn't support search.
   520	var ErrNoSearchRoot = errors.New("client: server doesn't support search")
   521	
   522	// ErrNoHelpRoot is returned by HelpRoot if the server doesn't have a help handler.
   523	var ErrNoHelpRoot = errors.New("client: server does not have a help handler")
   524	
   525	// ErrNoShareRoot is returned by ShareRoot if the server doesn't have a share handler.
   526	var ErrNoShareRoot = errors.New("client: server does not have a share handler")
   527	
   528	// ErrNoSigning is returned by ServerKeyID if the server doesn't support signing.
   529	var ErrNoSigning = fmt.Errorf("client: server doesn't support signing")
   530	
   531	// ErrNoStorageGeneration is returned by StorageGeneration if the
   532	// server doesn't report a storage generation value.
   533	var ErrNoStorageGeneration = errors.New("client: server doesn't report a storage generation")
   534	
   535	// ErrNoSync is returned by SyncHandlers if the server does not advertise syncs.
   536	var ErrNoSync = errors.New("client: server has no sync handlers")
   537	
   538	// BlobRoot returns the server's blobroot URL prefix.
   539	// If the client was constructed with an explicit path,
   540	// that path is used. Otherwise the server's
   541	// default advertised blobRoot is used.
   542	func (c *Client) BlobRoot() (string, error) {
   543		prefix, err := c.prefix()
   544		if err != nil {
   545			return "", err
   546		}
   547		return prefix + "/", nil
   548	}
   549	
   550	// ServerKeyID returns the server's GPG public key ID, in its long (16 capital
   551	// hex digits) format. If the server isn't running a sign handler, the error
   552	// will be ErrNoSigning.
   553	func (c *Client) ServerKeyID() (string, error) {
   554		if err := c.condDiscovery(); err != nil {
   555			return "", err
   556		}
   557		if c.serverKeyID == "" {
   558			return "", ErrNoSigning
   559		}
   560		return c.serverKeyID, nil
   561	}
   562	
   563	// ServerPublicKeyBlobRef returns the server's public key blobRef
   564	// If the server isn't running a sign handler, the error will be ErrNoSigning.
   565	func (c *Client) ServerPublicKeyBlobRef() (blob.Ref, error) {
   566		if err := c.condDiscovery(); err != nil {
   567			return blob.Ref{}, err
   568		}
   569	
   570		if !c.serverPublicKeyBlobRef.Valid() {
   571			return blob.Ref{}, ErrNoSigning
   572		}
   573		return c.serverPublicKeyBlobRef, nil
   574	}
   575	
   576	// SearchRoot returns the server's search handler.
   577	// If the server isn't running an index and search handler, the error
   578	// will be ErrNoSearchRoot.
   579	func (c *Client) SearchRoot() (string, error) {
   580		if err := c.condDiscovery(); err != nil {
   581			return "", err
   582		}
   583		if c.searchRoot == "" {
   584			return "", ErrNoSearchRoot
   585		}
   586		return c.searchRoot, nil
   587	}
   588	
   589	// HelpRoot returns the server's help handler.
   590	// If the server isn't running a help handler, the error will be
   591	// ErrNoHelpRoot.
   592	func (c *Client) HelpRoot() (string, error) {
   593		if err := c.condDiscovery(); err != nil {
   594			return "", err
   595		}
   596		if c.helpRoot == "" {
   597			return "", ErrNoHelpRoot
   598		}
   599		return c.helpRoot, nil
   600	}
   601	
   602	// ShareRoot returns the server's share handler prefix URL.
   603	// If the server isn't running a share handler, the error will be
   604	// ErrNoShareRoot.
   605	func (c *Client) ShareRoot() (string, error) {
   606		if err := c.condDiscovery(); err != nil {
   607			return "", err
   608		}
   609		if c.shareRoot == "" {
   610			return "", ErrNoShareRoot
   611		}
   612		return c.shareRoot, nil
   613	}
   614	
   615	// SignHandler returns the server's sign handler.
   616	// If the server isn't running a sign handler, the error will be
   617	// ErrNoSigning.
   618	func (c *Client) SignHandler() (string, error) {
   619		if err := c.condDiscovery(); err != nil {
   620			return "", err
   621		}
   622		if c.signHandler == "" {
   623			return "", ErrNoSigning
   624		}
   625		return c.signHandler, nil
   626	}
   627	
   628	// StorageGeneration returns the server's unique ID for its storage
   629	// generation, reset whenever storage is reset, moved, or partially
   630	// lost.
   631	//
   632	// This is a value that can be used in client cache keys to add
   633	// certainty that they're talking to the same instance as previously.
   634	//
   635	// If the server doesn't return such a value, the error will be
   636	// ErrNoStorageGeneration.
   637	func (c *Client) StorageGeneration() (string, error) {
   638		if err := c.condDiscovery(); err != nil {
   639			return "", err
   640		}
   641		if c.storageGen == "" {
   642			return "", ErrNoStorageGeneration
   643		}
   644		return c.storageGen, nil
   645	}
   646	
   647	// HasLegacySHA1 reports whether the server has SHA-1 blobs indexed.
   648	func (c *Client) HasLegacySHA1() (bool, error) {
   649		if err := c.condDiscovery(); err != nil {
   650			return false, err
   651		}
   652		return c.hasLegacySHA1, nil
   653	}
   654	
   655	// SyncInfo holds the data that were acquired with a discovery
   656	// and that are relevant to a syncHandler.
   657	type SyncInfo struct {
   658		From    string
   659		To      string
   660		ToIndex bool // whether this sync is from a blob storage to an index
   661	}
   662	
   663	// SyncHandlers returns the server's sync handlers "from" and
   664	// "to" prefix URLs.
   665	// If the server isn't running any sync handler, the error
   666	// will be ErrNoSync.
   667	func (c *Client) SyncHandlers() ([]*SyncInfo, error) {
   668		if err := c.condDiscovery(); err != nil {
   669			return nil, err
   670		}
   671		if c.syncHandlers == nil {
   672			return nil, ErrNoSync
   673		}
   674		return c.syncHandlers, nil
   675	}
   676	
   677	var _ search.GetRecentPermanoder = (*Client)(nil)
   678	
   679	// GetRecentPermanodes implements search.GetRecentPermanoder against a remote server over HTTP.
   680	func (c *Client) GetRecentPermanodes(ctx context.Context, req *search.RecentRequest) (*search.RecentResponse, error) {
   681		sr, err := c.SearchRoot()
   682		if err != nil {
   683			return nil, err
   684		}
   685		url := sr + req.URLSuffix()
   686		hreq := c.newRequest(ctx, "GET", url)
   687		hres, err := c.expect2XX(hreq)
   688		if err != nil {
   689			return nil, err
   690		}
   691		res := new(search.RecentResponse)
   692		if err := httputil.DecodeJSON(hres, res); err != nil {
   693			return nil, err
   694		}
   695		if err := res.Err(); err != nil {
   696			return nil, err
   697		}
   698		return res, nil
   699	}
   700	
   701	// GetPermanodesWithAttr searches for permanodes that match the given search request.
   702	// The Fuzzy option in the request must not be set, and the Attribute option
   703	// must be set.
   704	// Only indexed attributes may be queried.
   705	func (c *Client) GetPermanodesWithAttr(ctx context.Context, req *search.WithAttrRequest) (*search.WithAttrResponse, error) {
   706		sr, err := c.SearchRoot()
   707		if err != nil {
   708			return nil, err
   709		}
   710		url := sr + req.URLSuffix()
   711		hreq := c.newRequest(ctx, "GET", url)
   712		hres, err := c.expect2XX(hreq)
   713		if err != nil {
   714			return nil, err
   715		}
   716		res := new(search.WithAttrResponse)
   717		if err := httputil.DecodeJSON(hres, res); err != nil {
   718			return nil, err
   719		}
   720		if err := res.Err(); err != nil {
   721			return nil, err
   722		}
   723		return res, nil
   724	}
   725	
   726	func (c *Client) Describe(ctx context.Context, req *search.DescribeRequest) (*search.DescribeResponse, error) {
   727		sr, err := c.SearchRoot()
   728		if err != nil {
   729			return nil, err
   730		}
   731		url := sr + req.URLSuffixPost()
   732		body, err := json.MarshalIndent(req, "", "\t")
   733		if err != nil {
   734			return nil, err
   735		}
   736		hreq := c.newRequest(ctx, "POST", url, bytes.NewReader(body))
   737		hres, err := c.expect2XX(hreq)
   738		if err != nil {
   739			return nil, err
   740		}
   741		res := new(search.DescribeResponse)
   742		if err := httputil.DecodeJSON(hres, res); err != nil {
   743			return nil, err
   744		}
   745		return res, nil
   746	}
   747	
   748	func (c *Client) GetClaims(ctx context.Context, req *search.ClaimsRequest) (*search.ClaimsResponse, error) {
   749		sr, err := c.SearchRoot()
   750		if err != nil {
   751			return nil, err
   752		}
   753		url := sr + req.URLSuffix()
   754		hreq := c.newRequest(ctx, "GET", url)
   755		hres, err := c.expect2XX(hreq)
   756		if err != nil {
   757			return nil, err
   758		}
   759		res := new(search.ClaimsResponse)
   760		if err := httputil.DecodeJSON(hres, res); err != nil {
   761			return nil, err
   762		}
   763		return res, nil
   764	}
   765	
   766	func (c *Client) query(ctx context.Context, req *search.SearchQuery) (*http.Response, error) {
   767		sr, err := c.SearchRoot()
   768		if err != nil {
   769			return nil, err
   770		}
   771		url := sr + req.URLSuffix()
   772		body, err := json.Marshal(req)
   773		if err != nil {
   774			return nil, err
   775		}
   776		hreq := c.newRequest(ctx, "POST", url, bytes.NewReader(body))
   777		return c.expect2XX(hreq)
   778	}
   779	
   780	func (c *Client) Query(ctx context.Context, req *search.SearchQuery) (*search.SearchResult, error) {
   781		hres, err := c.query(ctx, req)
   782		if err != nil {
   783			return nil, err
   784		}
   785		res := new(search.SearchResult)
   786		if err := httputil.DecodeJSON(hres, res); err != nil {
   787			return nil, err
   788		}
   789		return res, nil
   790	}
   791	
   792	// QueryRaw sends req and returns the body of the response, which should be the
   793	// unparsed JSON of a search.SearchResult.
   794	func (c *Client) QueryRaw(ctx context.Context, req *search.SearchQuery) ([]byte, error) {
   795		hres, err := c.query(ctx, req)
   796		if err != nil {
   797			return nil, err
   798		}
   799		defer hres.Body.Close()
   800		return io.ReadAll(hres.Body)
   801	}
   802	
   803	// SearchExistingFileSchema does a search query looking for an
   804	// existing file with entire contents of wholeRef, then does a HEAD
   805	// request to verify the file still exists on the server. If so,
   806	// it returns that file schema's blobref.
   807	//
   808	// If multiple wholeRef values are provided, any may match. This is
   809	// used for searching for the file by multiple wholeRef hashes
   810	// (e.g. SHA-224 and SHA-1).
   811	//
   812	// It returns (zero, nil) if not found. A non-nil error is only returned
   813	// if there were problems searching.
   814	func (c *Client) SearchExistingFileSchema(ctx context.Context, wholeRef ...blob.Ref) (blob.Ref, error) {
   815		sr, err := c.SearchRoot()
   816		if err != nil {
   817			return blob.Ref{}, err
   818		}
   819		if len(wholeRef) == 0 {
   820			return blob.Ref{}, nil
   821		}
   822		url := sr + "camli/search/files"
   823		for i, ref := range wholeRef {
   824			if i == 0 {
   825				url += "?wholedigest=" + ref.String()
   826			} else {
   827				url += "&wholedigest=" + ref.String()
   828			}
   829		}
   830		req := c.newRequest(ctx, "GET", url)
   831		res, err := c.doReqGated(req)
   832		if err != nil {
   833			return blob.Ref{}, err
   834		}
   835		if res.StatusCode != 200 {
   836			body, _ := io.ReadAll(io.LimitReader(res.Body, 1<<20))
   837			res.Body.Close()
   838			return blob.Ref{}, fmt.Errorf("client: got status code %d from URL %s; body %s", res.StatusCode, url, body)
   839		}
   840		var ress camtypes.FileSearchResponse
   841		if err := httputil.DecodeJSON(res, &ress); err != nil {
   842			// Check that we're not just hitting the change introduced in 2018-01-13-6e8a5930c9fee81640c6c75a9a549fec98064186
   843			mismatch, err := c.versionMismatch(ctx)
   844			if err != nil {
   845				log.Printf("Could not verify whether client is too recent or server is too old: %v", err)
   846			} else if mismatch {
   847				return blob.Ref{}, fmt.Errorf("Client is too recent for this server. Use a client built before 2018-01-13-6e8a5930c9, or upgrade the server to after that revision.")
   848			}
   849			return blob.Ref{}, fmt.Errorf("client: error parsing JSON from URL %s: %w", url, err)
   850		}
   851		if len(ress.Files) == 0 {
   852			return blob.Ref{}, nil
   853		}
   854		for wholeRef, files := range ress.Files {
   855			for _, f := range files {
   856				if c.FileHasContents(ctx, f, blob.MustParse(wholeRef)) {
   857					return f, nil
   858				}
   859			}
   860		}
   861		return blob.Ref{}, nil
   862	}
   863	
   864	// versionMismatch returns true if the server was built before 2018-01-13 and
   865	// the client was built at or after 2018-01-13.
   866	func (c *Client) versionMismatch(ctx context.Context) (bool, error) {
   867		const shortRFC3339 = "2006-01-02"
   868		version := buildinfo.GitInfo
   869		if version == "" {
   870			return false, errors.New("unknown client version")
   871		}
   872		version = version[:10] // keep only the date part
   873		clientDate, err := time.Parse(shortRFC3339, version)
   874		if err != nil {
   875			return false, fmt.Errorf("could not parse date from version %q: %w", version, err)
   876		}
   877		apiChangeDate, _ := time.Parse(shortRFC3339, "2018-01-13")
   878		if !clientDate.After(apiChangeDate) {
   879			// client is old enough, all good.
   880			return false, nil
   881		}
   882		url := c.discoRoot() + "/status/status.json"
   883		req := c.newRequest(ctx, "GET", url)
   884		res, err := c.doReqGated(req)
   885		if err != nil {
   886			return false, err
   887		}
   888		if res.StatusCode != 200 {
   889			body, _ := io.ReadAll(io.LimitReader(res.Body, 1<<20))
   890			res.Body.Close()
   891			return false, fmt.Errorf("got status code %d from URL %s; body %s", res.StatusCode, url, body)
   892		}
   893		var status struct {
   894			Version string `json:"version"`
   895		}
   896		if err := httputil.DecodeJSON(res, &status); err != nil {
   897			return false, fmt.Errorf("error parsing JSON from URL %s: %w", url, err)
   898		}
   899		serverVersion := status.Version[:10]
   900		serverDate, err := time.Parse(shortRFC3339, serverVersion)
   901		if err != nil {
   902			return false, fmt.Errorf("could not parse date from server version %q: %w", status.Version, err)
   903		}
   904		if serverDate.After(apiChangeDate) {
   905			// server is recent enough, all good.
   906			return false, nil
   907		}
   908		return true, nil
   909	}
   910	
   911	// FileHasContents returns true iff f refers to a "file" or "bytes" schema blob,
   912	// the server is configured with a "download helper", and the server responds
   913	// that all chunks of 'f' are available and match the digest of wholeRef.
   914	func (c *Client) FileHasContents(ctx context.Context, f, wholeRef blob.Ref) bool {
   915		if err := c.condDiscovery(); err != nil {
   916			return false
   917		}
   918		if c.downloadHelper == "" {
   919			return false
   920		}
   921		req := c.newRequest(ctx, "HEAD", c.downloadHelper+f.String()+"/?verifycontents="+wholeRef.String())
   922		res, err := c.expect2XX(req)
   923		if err != nil {
   924			log.Printf("download helper HEAD error: %v", err)
   925			return false
   926		}
   927		defer res.Body.Close()
   928		return res.Header.Get("X-Camli-Contents") == wholeRef.String()
   929	}
   930	
   931	// prefix returns the URL prefix before "/camli/", or before
   932	// the blobref hash in case of a share URL.
   933	// Examples: http://foo.com:3179/bs or http://foo.com:3179/share
   934	func (c *Client) prefix() (string, error) {
   935		if err := c.prefixOnce.Do(c.initPrefix); err != nil {
   936			return "", err
   937		}
   938		return c.prefixv, nil
   939	}
   940	
   941	// blobPrefix returns the URL prefix before the blobref hash.
   942	// Example: http://foo.com:3179/bs/camli or http://foo.com:3179/share
   943	func (c *Client) blobPrefix() (string, error) {
   944		pfx, err := c.prefix()
   945		if err != nil {
   946			return "", err
   947		}
   948		if !c.isSharePrefix {
   949			pfx += "/camli"
   950		}
   951		return pfx, nil
   952	}
   953	
   954	// discoRoot returns the user defined server for this client. It prepends "https://" if no scheme was specified.
   955	func (c *Client) discoRoot() string {
   956		s := c.server
   957		if c.sameOrigin {
   958			s = strings.TrimPrefix(s, "http://")
   959			s = strings.TrimPrefix(s, "https://")
   960			parts := strings.SplitN(s, "/", 1)
   961			if len(parts) < 2 {
   962				return "/"
   963			}
   964			return "/" + parts[1]
   965		}
   966		if !strings.HasPrefix(s, "http") {
   967			s = "https://" + s
   968		}
   969		return s
   970	}
   971	
   972	// initPrefix uses the user provided server URL to define the URL
   973	// prefix to the blobserver root. If the server URL has a path
   974	// component then it is directly used, otherwise the blobRoot
   975	// from the discovery is used as the path.
   976	func (c *Client) initPrefix() error {
   977		c.isSharePrefix = false
   978		root := c.discoRoot()
   979		u, err := url.Parse(root)
   980		if err != nil {
   981			return err
   982		}
   983		if len(u.Path) > 1 {
   984			c.prefixv = strings.TrimRight(root, "/")
   985			return nil
   986		}
   987		return c.condDiscovery()
   988	}
   989	
   990	func (c *Client) condDiscovery() error {
   991		if c.sto != nil {
   992			return errors.New("client not using HTTP")
   993		}
   994		return c.discoOnce.Do(c.doDiscovery)
   995	}
   996	
   997	// DiscoveryDoc returns the server's JSON discovery document.
   998	// This method exists purely for the "camtool discovery" command.
   999	// Clients shouldn't have to parse this themselves.
  1000	func (c *Client) DiscoveryDoc(ctx context.Context) (io.Reader, error) {
  1001		res, err := c.discoveryResp(ctx)
  1002		if err != nil {
  1003			return nil, err
  1004		}
  1005		defer res.Body.Close()
  1006		const maxSize = 1 << 20
  1007		all, err := io.ReadAll(io.LimitReader(res.Body, maxSize+1))
  1008		if err != nil {
  1009			return nil, err
  1010		}
  1011		if len(all) > maxSize {
  1012			return nil, errors.New("discovery document oddly large")
  1013		}
  1014		if len(all) > 0 && all[len(all)-1] != '\n' {
  1015			all = append(all, '\n')
  1016		}
  1017		return bytes.NewReader(all), err
  1018	}
  1019	
  1020	// HTTPVersion reports the HTTP version in use, such as "HTTP/1.1" or "HTTP/2.0".
  1021	func (c *Client) HTTPVersion(ctx context.Context) (string, error) {
  1022		req := c.newRequest(ctx, "HEAD", c.discoRoot(), nil)
  1023		res, err := c.doReqGated(req)
  1024		if err != nil {
  1025			return "", err
  1026		}
  1027		return res.Proto, err
  1028	}
  1029	
  1030	func (c *Client) discoveryResp(ctx context.Context) (*http.Response, error) {
  1031		// If the path is just "" or "/", do discovery against
  1032		// the URL to see which path we should actually use.
  1033		req := c.newRequest(ctx, "GET", c.discoRoot(), nil)
  1034		req.Header.Set("Accept", "text/x-camli-configuration")
  1035		res, err := c.doReqGated(req)
  1036		if err != nil {
  1037			return nil, err
  1038		}
  1039		if res.StatusCode != 200 {
  1040			res.Body.Close()
  1041			errMsg := fmt.Sprintf("got status %q from blobserver URL %q during configuration discovery", res.Status, c.discoRoot())
  1042			if res.StatusCode == http.StatusUnauthorized && c.authErr != nil {
  1043				errMsg = fmt.Sprintf("%v. %v", c.authErr, errMsg)
  1044			}
  1045			return nil, errors.New(errMsg)
  1046		}
  1047		// TODO(bradfitz): little weird in retrospect that we request
  1048		// text/x-camli-configuration and expect to get back
  1049		// text/javascript.  Make them consistent.
  1050		if ct := res.Header.Get("Content-Type"); ct != "text/javascript" {
  1051			res.Body.Close()
  1052			return nil, fmt.Errorf("Blobserver returned unexpected type %q from discovery", ct)
  1053		}
  1054		return res, nil
  1055	}
  1056	
  1057	func (c *Client) doDiscovery() error {
  1058		ctx := context.TODO()
  1059		root, err := url.Parse(c.discoRoot())
  1060		if err != nil {
  1061			return err
  1062		}
  1063	
  1064		res, err := c.discoveryResp(ctx)
  1065		if err != nil {
  1066			return err
  1067		}
  1068	
  1069		var disco camtypes.Discovery
  1070		if err := httputil.DecodeJSON(res, &disco); err != nil {
  1071			return err
  1072		}
  1073	
  1074		if disco.SearchRoot == "" {
  1075			c.searchRoot = ""
  1076		} else {
  1077			u, err := root.Parse(disco.SearchRoot)
  1078			if err != nil {
  1079				return fmt.Errorf("client: invalid searchRoot %q; failed to resolve", disco.SearchRoot)
  1080			}
  1081			c.searchRoot = u.String()
  1082		}
  1083	
  1084		u, err := root.Parse(disco.HelpRoot)
  1085		if err != nil {
  1086			return fmt.Errorf("client: invalid helpRoot %q; failed to resolve", disco.HelpRoot)
  1087		}
  1088		c.helpRoot = u.String()
  1089	
  1090		u, err = root.Parse(disco.ShareRoot)
  1091		if err != nil {
  1092			return fmt.Errorf("client: invalid shareRoot %q; failed to resolve", disco.ShareRoot)
  1093		}
  1094		c.shareRoot = u.String()
  1095	
  1096		c.storageGen = disco.StorageGeneration
  1097		c.hasLegacySHA1 = disco.HasLegacySHA1Index
  1098	
  1099		u, err = root.Parse(disco.BlobRoot)
  1100		if err != nil {
  1101			return fmt.Errorf("client: error resolving blobRoot: %w", err)
  1102		}
  1103		c.prefixv = strings.TrimRight(u.String(), "/")
  1104	
  1105		if disco.UIDiscovery != nil {
  1106			u, err = root.Parse(disco.DownloadHelper)
  1107			if err != nil {
  1108				return fmt.Errorf("client: invalid downloadHelper %q; failed to resolve", disco.DownloadHelper)
  1109			}
  1110			c.downloadHelper = u.String()
  1111		}
  1112	
  1113		if disco.SyncHandlers != nil {
  1114			for _, v := range disco.SyncHandlers {
  1115				ufrom, err := root.Parse(v.From)
  1116				if err != nil {
  1117					return fmt.Errorf("client: invalid %q \"from\" sync; failed to resolve", v.From)
  1118				}
  1119				uto, err := root.Parse(v.To)
  1120				if err != nil {
  1121					return fmt.Errorf("client: invalid %q \"to\" sync; failed to resolve", v.To)
  1122				}
  1123				c.syncHandlers = append(c.syncHandlers, &SyncInfo{
  1124					From:    ufrom.String(),
  1125					To:      uto.String(),
  1126					ToIndex: v.ToIndex,
  1127				})
  1128			}
  1129		}
  1130	
  1131		if disco.Signing != nil {
  1132			c.serverKeyID = disco.Signing.PublicKeyID
  1133			c.serverPublicKeyBlobRef = disco.Signing.PublicKeyBlobRef
  1134			c.signHandler = disco.Signing.SignHandler
  1135		}
  1136		return nil
  1137	}
  1138	
  1139	// GetJSON sends a GET request to url, and unmarshals the returned
  1140	// JSON response into data. The URL's host must match the client's
  1141	// configured server.
  1142	func (c *Client) GetJSON(ctx context.Context, url string, data interface{}) error {
  1143		if !strings.HasPrefix(url, c.discoRoot()) {
  1144			return fmt.Errorf("wrong URL (%q) for this server", url)
  1145		}
  1146		hreq := c.newRequest(ctx, "GET", url)
  1147		resp, err := c.expect2XX(hreq)
  1148		if err != nil {
  1149			return err
  1150		}
  1151		return httputil.DecodeJSON(resp, data)
  1152	}
  1153	
  1154	// Post is like http://golang.org/pkg/net/http/#Client.Post
  1155	// but with implementation details like gated requests. The
  1156	// URL's host must match the client's configured server.
  1157	func (c *Client) Post(ctx context.Context, url string, bodyType string, body io.Reader) error {
  1158		resp, err := c.post(ctx, url, bodyType, body)
  1159		if err != nil {
  1160			return err
  1161		}
  1162		return resp.Body.Close()
  1163	}
  1164	
  1165	// Sign sends a request to the sign handler on server to sign the contents of r,
  1166	// and return them signed. It uses the same implementation details, such as gated
  1167	// requests, as Post.
  1168	func (c *Client) Sign(ctx context.Context, server string, r io.Reader) (signed []byte, err error) {
  1169		signHandler, err := c.SignHandler()
  1170		if err != nil {
  1171			return nil, err
  1172		}
  1173		signServer := strings.TrimSuffix(server, "/") + signHandler
  1174		resp, err := c.post(ctx, signServer, "application/x-www-form-urlencoded", r)
  1175		if err != nil {
  1176			return nil, err
  1177		}
  1178		defer resp.Body.Close()
  1179		return io.ReadAll(resp.Body)
  1180	}
  1181	
  1182	func (c *Client) post(ctx context.Context, url string, bodyType string, body io.Reader) (*http.Response, error) {
  1183		if !c.sameOrigin && !strings.HasPrefix(url, c.discoRoot()) {
  1184			return nil, fmt.Errorf("wrong URL (%q) for this server", url)
  1185		}
  1186		req := c.newRequest(ctx, "POST", url, body)
  1187		req.Header.Set("Content-Type", bodyType)
  1188		res, err := c.expect2XX(req)
  1189		if err != nil {
  1190			return nil, err
  1191		}
  1192		return res, nil
  1193	}
  1194	
  1195	// newRequest creates a request with the authentication header, and with the
  1196	// appropriate scheme and port in the case of self-signed TLS.
  1197	func (c *Client) newRequest(ctx context.Context, method, url string, body ...io.Reader) *http.Request {
  1198		var bodyR io.Reader
  1199		if len(body) > 0 {
  1200			bodyR = body[0]
  1201		}
  1202		if len(body) > 1 {
  1203			panic("too many body arguments")
  1204		}
  1205		req, err := http.NewRequest(method, url, bodyR)
  1206		if err != nil {
  1207			panic(err.Error())
  1208		}
  1209		// not done by http.NewRequest in Go 1.0:
  1210		if br, ok := bodyR.(*bytes.Reader); ok {
  1211			req.ContentLength = int64(br.Len())
  1212		}
  1213		c.authMode.AddAuthHeader(req)
  1214		return req.WithContext(ctx)
  1215	}
  1216	
  1217	// expect2XX will doReqGated and promote HTTP response codes outside of
  1218	// the 200-299 range to a non-nil error containing the response body.
  1219	func (c *Client) expect2XX(req *http.Request) (*http.Response, error) {
  1220		res, err := c.doReqGated(req)
  1221		if err == nil && (res.StatusCode < 200 || res.StatusCode > 299) {
  1222			buf := new(bytes.Buffer)
  1223			io.CopyN(buf, res.Body, 1<<20)
  1224			res.Body.Close()
  1225			return res, fmt.Errorf("client: got status code %d from URL %s; body %s", res.StatusCode, req.URL.String(), buf.String())
  1226		}
  1227		return res, err
  1228	}
  1229	
  1230	func (c *Client) doReqGated(req *http.Request) (*http.Response, error) {
  1231		c.httpGate.Start()
  1232		defer c.httpGate.Done()
  1233		return c.httpClient.Do(req)
  1234	}
  1235	
  1236	// DialFunc returns the adequate dial function when we're on android.
  1237	func (c *Client) DialFunc() func(network, addr string) (net.Conn, error) {
  1238		if c.useTLS() {
  1239			return nil
  1240		}
  1241		if android.IsChild() {
  1242			return func(network, addr string) (net.Conn, error) {
  1243				return android.Dial(network, addr)
  1244			}
  1245		}
  1246		return nil
  1247	}
  1248	
  1249	func (c *Client) http2DialTLSFunc() func(network, addr string, cfg *tls.Config) (net.Conn, error) {
  1250		trustedCerts := c.getTrustedCerts()
  1251		if !c.insecureAnyTLSCert && len(trustedCerts) == 0 {
  1252			// TLS with normal/full verification.
  1253			// nil means http2 uses its default dialer.
  1254			return nil
  1255		}
  1256		return func(network, addr string, cfg *tls.Config) (net.Conn, error) {
  1257			// we own cfg, so we can mutate it:
  1258			cfg.InsecureSkipVerify = true
  1259			conn, err := tls.Dial(network, addr, cfg)
  1260			if err != nil {
  1261				return nil, err
  1262			}
  1263			if c.insecureAnyTLSCert {
  1264				return conn, err
  1265			}
  1266			state := conn.ConnectionState()
  1267			if p := state.NegotiatedProtocol; p != http2.NextProtoTLS {
  1268				return nil, fmt.Errorf("http2: unexpected ALPN protocol %q; want %q", p, http2.NextProtoTLS)
  1269			}
  1270			certs := state.PeerCertificates
  1271			if len(certs) < 1 {
  1272				return nil, fmt.Errorf("no TLS peer certificates from %s", addr)
  1273			}
  1274			sig := hashutil.SHA256Prefix(certs[0].Raw)
  1275			if slices.Contains(trustedCerts, sig) {
  1276				return conn, nil
  1277			}
  1278			return nil, fmt.Errorf("TLS server at %v presented untrusted certificate (signature %q)", addr, sig)
  1279		}
  1280	}
  1281	
  1282	// DialTLSFunc returns the adequate dial function, when using SSL, depending on
  1283	// whether we're using insecure TLS (certificate verification is disabled), or we
  1284	// have some trusted certs, or we're on android.
  1285	// If the client's config has some trusted certs, the server's certificate will
  1286	// be checked against those in the config after the TLS handshake.
  1287	func (c *Client) DialTLSFunc() func(network, addr string) (net.Conn, error) {
  1288		if !c.useTLS() {
  1289			return nil
  1290		}
  1291		trustedCerts := c.getTrustedCerts()
  1292		var stdTLS bool
  1293		if !c.insecureAnyTLSCert && len(trustedCerts) == 0 {
  1294			// TLS with normal/full verification.
  1295			stdTLS = true
  1296			if !android.IsChild() {
  1297				// Not android, so let the stdlib deal with it
  1298				return nil
  1299			}
  1300		}
  1301	
  1302		return func(network, addr string) (net.Conn, error) {
  1303			var conn *tls.Conn
  1304			var err error
  1305			if android.IsChild() {
  1306				ac, err := android.Dial(network, addr)
  1307				if err != nil {
  1308					return nil, err
  1309				}
  1310				var tlsConfig *tls.Config
  1311				if stdTLS {
  1312					tlsConfig, err = android.TLSConfig()
  1313					if err != nil {
  1314						return nil, err
  1315					}
  1316				} else {
  1317					tlsConfig = &tls.Config{InsecureSkipVerify: true}
  1318				}
  1319				// Since we're doing the TLS handshake ourselves, we need to set the ServerName,
  1320				// in case the server uses SNI (as is the case if it's relying on Let's Encrypt,
  1321				// for example).
  1322				tlsConfig.ServerName = c.serverNameOfAddr(addr)
  1323				conn = tls.Client(ac, tlsConfig)
  1324				if err := conn.Handshake(); err != nil {
  1325					return nil, err
  1326				}
  1327				if stdTLS {
  1328					// Normal TLS verification succeeded and we do not have
  1329					// additional trusted certificate fingerprints to check for.
  1330					return conn, nil
  1331				}
  1332			} else {
  1333				conn, err = tls.Dial(network, addr, &tls.Config{InsecureSkipVerify: true})
  1334				if err != nil {
  1335					return nil, err
  1336				}
  1337			}
  1338			if c.insecureAnyTLSCert {
  1339				return conn, nil
  1340			}
  1341			certs := conn.ConnectionState().PeerCertificates
  1342			if len(certs) < 1 {
  1343				return nil, fmt.Errorf("no TLS peer certificates from %s", addr)
  1344			}
  1345			sig := hashutil.SHA256Prefix(certs[0].Raw)
  1346			if slices.Contains(trustedCerts, sig) {
  1347				return conn, nil
  1348			}
  1349			return nil, fmt.Errorf("TLS server at %v presented untrusted certificate (signature %q)", addr, sig)
  1350		}
  1351	}
  1352	
  1353	// serverNameOfAddr returns the host part of addr, or the empty string if addr
  1354	// is not a valid address (see net.Dial). Additionally, if host is an IP literal,
  1355	// serverNameOfAddr returns the empty string.
  1356	func (c *Client) serverNameOfAddr(addr string) string {
  1357		serverName, _, err := net.SplitHostPort(addr)
  1358		if err != nil {
  1359			c.printf("could not get server name from address %q: %v", addr, err)
  1360			return ""
  1361		}
  1362		if ip := net.ParseIP(serverName); ip != nil {
  1363			return ""
  1364		}
  1365		return serverName
  1366	}
  1367	
  1368	// Signer returns the client's Signer, if any. The Signer signs JSON
  1369	// mutation claims.
  1370	func (c *Client) Signer() (*schema.Signer, error) {
  1371		c.signerOnce.Do(c.signerInit)
  1372		return c.signer, c.signerErr
  1373	}
  1374	
  1375	func (c *Client) signerInit() {
  1376		c.signer, c.signerErr = c.buildSigner()
  1377	}
  1378	
  1379	func (c *Client) buildSigner() (*schema.Signer, error) {
  1380		c.initSignerPublicKeyBlobrefOnce.Do(c.initSignerPublicKeyBlobref)
  1381		if !c.signerPublicKeyRef.Valid() {
  1382			return nil, camtypes.ErrClientNoPublicKey
  1383		}
  1384		return schema.NewSigner(c.signerPublicKeyRef, strings.NewReader(c.publicKeyArmored), c.SecretRingFile())
  1385	}
  1386	
  1387	// sigTime optionally specifies the signature time.
  1388	// If zero, the current time is used.
  1389	func (c *Client) signBlob(ctx context.Context, bb schema.Buildable, sigTime time.Time) (string, error) {
  1390		signer, err := c.Signer()
  1391		if err != nil {
  1392			return "", err
  1393		}
  1394		return bb.Builder().SignAt(ctx, signer, sigTime)
  1395	}
  1396	
  1397	// UploadPublicKey uploads the public key (if one is defined), so
  1398	// subsequent (likely synchronous) indexing of uploaded signed blobs
  1399	// will have access to the public key to verify it. In the normal
  1400	// case, the stat cache prevents this from doing anything anyway.
  1401	func (c *Client) UploadPublicKey(ctx context.Context) error {
  1402		sigRef := c.SignerPublicKeyBlobref()
  1403		if !sigRef.Valid() {
  1404			return nil
  1405		}
  1406		var err error
  1407		if _, keyUploaded := c.haveCache.StatBlobCache(sigRef); !keyUploaded {
  1408			_, err = c.uploadString(ctx, c.publicKeyArmored, false)
  1409		}
  1410		return err
  1411	}
  1412	
  1413	// checkMatchingKeys compares the client's and the server's keys and logs if they differ.
  1414	func (c *Client) checkMatchingKeys() {
  1415		serverKey, err := c.ServerKeyID()
  1416		if err != nil {
  1417			log.Printf("Warning: Could not obtain ther server's key id: %v", err)
  1418			return
  1419		}
  1420		if serverKey != c.signer.KeyIDLong() {
  1421			log.Printf("Warning: client (%s) and server (%s) keys differ.", c.signer.KeyIDLong(), serverKey)
  1422		}
  1423	}
  1424	
  1425	func (c *Client) UploadAndSignBlob(ctx context.Context, b schema.AnyBlob) (*PutResult, error) {
  1426		signed, err := c.signBlob(ctx, b.Blob(), time.Time{})
  1427		if err != nil {
  1428			return nil, err
  1429		}
  1430		c.checkMatchingKeys()
  1431		if err := c.UploadPublicKey(ctx); err != nil {
  1432			return nil, err
  1433		}
  1434		return c.uploadString(ctx, signed, false)
  1435	}
  1436	
  1437	func (c *Client) UploadBlob(ctx context.Context, b schema.AnyBlob) (*PutResult, error) {
  1438		// TODO(bradfitz): ask the blob for its own blobref, rather
  1439		// than changing the hash function with uploadString?
  1440		return c.uploadString(ctx, b.Blob().JSON(), true)
  1441	}
  1442	
  1443	func (c *Client) uploadString(ctx context.Context, s string, stat bool) (*PutResult, error) {
  1444		uh := NewUploadHandleFromString(s)
  1445		uh.SkipStat = !stat
  1446		return c.Upload(ctx, uh)
  1447	}
  1448	
  1449	func (c *Client) UploadNewPermanode(ctx context.Context) (*PutResult, error) {
  1450		unsigned := schema.NewUnsignedPermanode()
  1451		return c.UploadAndSignBlob(ctx, unsigned)
  1452	}
  1453	
  1454	func (c *Client) UploadPlannedPermanode(ctx context.Context, key string, sigTime time.Time) (*PutResult, error) {
  1455		unsigned := schema.NewPlannedPermanode(key)
  1456		signed, err := c.signBlob(ctx, unsigned, sigTime)
  1457		if err != nil {
  1458			return nil, err
  1459		}
  1460		c.checkMatchingKeys()
  1461		if err := c.UploadPublicKey(ctx); err != nil {
  1462			return nil, err
  1463		}
  1464		return c.uploadString(ctx, signed, true)
  1465	}
  1466	
  1467	// IsIgnoredFile returns whether the file at fullpath should be ignored by pk-put.
  1468	// The fullpath is checked against the ignoredFiles list, trying the following rules in this order:
  1469	// 1) star-suffix style matching (.e.g *.jpg).
  1470	// 2) Shell pattern match as done by http://golang.org/pkg/path/filepath/#Match
  1471	// 3) If the pattern is an absolute path to a directory, fullpath matches if it is that directory or a child of it.
  1472	// 4) If the pattern is a relative path, fullpath matches if it has pattern as a path component (i.e the pattern is a part of fullpath that fits exactly between two path separators).
  1473	func (c *Client) IsIgnoredFile(fullpath string) bool {
  1474		c.initIgnoredFilesOnce.Do(c.initIgnoredFiles)
  1475		return c.ignoreChecker(fullpath)
  1476	}
  1477	
  1478	// Close closes the client.
  1479	func (c *Client) Close() error {
  1480		if cl, ok := c.sto.(io.Closer); ok {
  1481			return cl.Close()
  1482		}
  1483		if c := c.HTTPClient(); c != nil {
  1484			switch t := c.Transport.(type) {
  1485			case *http.Transport:
  1486				t.CloseIdleConnections()
  1487			case *http2.Transport:
  1488				t.CloseIdleConnections()
  1489			}
  1490		}
  1491		return nil
  1492	}
  1493	
  1494	func httpGateSize(rt http.RoundTripper) int {
  1495		switch v := rt.(type) {
  1496		case *httputil.StatsTransport:
  1497			return httpGateSize(v.Transport)
  1498		case *http.Transport:
  1499			return maxParallelHTTP_h1
  1500		case *http2.Transport:
  1501			return maxParallelHTTP_h2
  1502		default:
  1503			return maxParallelHTTP_h1 // conservative default
  1504		}
  1505	}