1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
16
17
18 package client
19
20 import (
21 "bytes"
22 "context"
23 "crypto/tls"
24 "encoding/json"
25 "errors"
26 "fmt"
27 "io"
28 "io/ioutil"
29 "log"
30 "net"
31 "net/http"
32 "net/url"
33 "os"
34 "regexp"
35 "strings"
36 "sync"
37 "time"
38
39 "perkeep.org/internal/hashutil"
40 "perkeep.org/internal/httputil"
41 "perkeep.org/internal/osutil"
42 "perkeep.org/pkg/auth"
43 "perkeep.org/pkg/blob"
44 "perkeep.org/pkg/blobserver"
45 "perkeep.org/pkg/buildinfo"
46 "perkeep.org/pkg/client/android"
47 "perkeep.org/pkg/schema"
48 "perkeep.org/pkg/search"
49 "perkeep.org/pkg/types/camtypes"
50
51 "go4.org/syncutil"
52 "golang.org/x/net/http2"
53 )
54
55
56
57
58
59 type Client struct {
60
61
62
63
64
65
66 server string
67
68 prefixOnce syncutil.Once
69 prefixv string
70 isSharePrefix bool
71
72 discoOnce syncutil.Once
73 searchRoot string
74 downloadHelper string
75 storageGen string
76 hasLegacySHA1 bool
77 syncHandlers []*SyncInfo
78 serverKeyID string
79 helpRoot string
80 shareRoot string
81 serverPublicKeyBlobRef blob.Ref
82
83 signerOnce sync.Once
84 signer *schema.Signer
85 signerErr error
86 signHandler string
87
88 authMode auth.AuthMode
89
90
91 authErr error
92
93 httpClient *http.Client
94 haveCache HaveCache
95
96
97 sto blobserver.Storage
98
99 initTrustedCertsOnce sync.Once
100
101
102
103
104
105
106
107
108
109
110 trustedCerts []string
111
112
113
114 insecureAnyTLSCert bool
115
116 initIgnoredFilesOnce sync.Once
117
118
119
120
121 ignoredFiles []string
122 ignoreChecker func(path string) bool
123
124 initSignerPublicKeyBlobrefOnce sync.Once
125 signerPublicKeyRef blob.Ref
126 publicKeyArmored string
127
128 statsMutex sync.Mutex
129 stats Stats
130
131
132
133
134 viaMu sync.RWMutex
135 via map[blob.Ref]blob.Ref
136
137
138
139
140 Verbose bool
141
142
143
144
145 Logger *log.Logger
146
147 httpGate *syncutil.Gate
148 transportConfig *TransportConfig
149
150 noExtConfig bool
151
152
153
154
155
156 sameOrigin bool
157 }
158
159 const maxParallelHTTP_h1 = 5
160 const maxParallelHTTP_h2 = 50
161
162
163
164 var inGopherJS bool
165
166
167
168
169
170 func New(opts ...ClientOption) (*Client, error) {
171 c := &Client{
172 haveCache: noHaveCache{},
173 Logger: log.New(os.Stderr, "", log.Ldate|log.Ltime),
174 authMode: auth.None{},
175 }
176 for _, v := range opts {
177 v.modifyClient(c)
178 }
179 if c.sto != nil && len(opts) > 1 {
180 return nil, errors.New("use of OptionUseStorageClient precludes use of any other options")
181 }
182
183 if inGopherJS {
184 c.noExtConfig = true
185 c.sameOrigin = true
186 }
187 if c.noExtConfig {
188 c.setDefaultHTTPClient()
189 return c, nil
190 }
191
192 if c.server != "" {
193 if !isURLOrHostPort(c.server) {
194 configOnce.Do(parseConfig)
195 serverConf, ok := config.Servers[c.server]
196 if !ok {
197 log.Fatalf("%q looks like a server alias, but no such alias found in config at %v", c.server, osutil.UserClientConfigPath())
198 }
199 c.server = serverConf.Server
200 }
201 c.setDefaultHTTPClient()
202 return c, nil
203 }
204
205 var err error
206 c.server, err = getServer()
207 if err != nil {
208 return nil, err
209 }
210 err = c.SetupAuth()
211 if err != nil {
212 return nil, err
213 }
214 c.setDefaultHTTPClient()
215 return c, nil
216 }
217
218 func (c *Client) setDefaultHTTPClient() {
219 if c.httpClient == nil {
220 c.httpClient = &http.Client{
221 Transport: c.transportForConfig(c.transportConfig),
222 }
223 }
224 c.httpGate = syncutil.NewGate(httpGateSize(c.httpClient.Transport))
225 }
226
227
228 func NewOrFail(opts ...ClientOption) *Client {
229 c, err := New(opts...)
230 if err != nil {
231 log.Fatalf("error creating client: %v", err)
232 }
233 return c
234 }
235
236
237 func (c *Client) NewPathClient(path string) (*Client, error) {
238 u, err := url.Parse(c.server)
239 if err != nil {
240 return nil, fmt.Errorf("bogus server %q for NewPathClient receiver: %v", c.server, err)
241 }
242 u.Path = path
243 pc, err := New(OptionServer(u.String()))
244 if err != nil {
245 return nil, err
246 }
247 pc.authMode = c.authMode
248 pc.discoOnce.Do(noop)
249 return pc, nil
250 }
251
252
253 type TransportConfig struct {
254
255
256 Proxy func(*http.Request) (*url.URL, error)
257 Verbose bool
258 }
259
260 func (c *Client) useHTTP2(tc *TransportConfig) bool {
261 if !c.useTLS() {
262 return false
263 }
264 if android.IsChild() {
265
266 return false
267 }
268 if os.Getenv("HTTPS_PROXY") != "" || os.Getenv("https_proxy") != "" ||
269 (tc != nil && tc.Proxy != nil) {
270
271 return false
272 }
273 return true
274 }
275
276
277
278
279
280 func (c *Client) transportForConfig(tc *TransportConfig) http.RoundTripper {
281 if inGopherJS {
282
283
284
285
286 return nil
287 }
288 if c == nil {
289 return nil
290 }
291 var transport http.RoundTripper
292 proxy := http.ProxyFromEnvironment
293 if tc != nil && tc.Proxy != nil {
294 proxy = tc.Proxy
295 }
296
297 if c.useHTTP2(tc) {
298 transport = &http2.Transport{
299 DialTLS: c.http2DialTLSFunc(),
300 }
301 } else {
302 transport = &http.Transport{
303 DialTLS: c.DialTLSFunc(),
304 Dial: c.DialFunc(),
305 Proxy: proxy,
306 MaxIdleConnsPerHost: maxParallelHTTP_h1,
307 }
308 }
309 httpStats := &httputil.StatsTransport{
310 Transport: transport,
311 }
312 if tc != nil {
313 httpStats.VerboseLog = tc.Verbose
314 }
315 transport = httpStats
316 if android.IsChild() {
317 transport = &android.StatsTransport{Rt: transport}
318 }
319 return transport
320 }
321
322
323
324 func (c *Client) HTTPStats() *httputil.StatsTransport {
325 st, _ := c.httpClient.Transport.(*httputil.StatsTransport)
326 return st
327 }
328
329 type ClientOption interface {
330 modifyClient(*Client)
331 }
332
333
334
335
336
337
338
339
340 func OptionServer(server string) ClientOption {
341 return optionServer(server)
342 }
343
344 type optionServer string
345
346 func (s optionServer) modifyClient(c *Client) { c.server = string(s) }
347
348
349
350
351
352
353
354
355
356
357
358 func OptionUseStorageClient(s blobserver.Storage) ClientOption {
359 return optionStorage{s}
360 }
361
362 type optionStorage struct {
363 sto blobserver.Storage
364 }
365
366 func (o optionStorage) modifyClient(c *Client) {
367 c.sto = o.sto
368 c.noExtConfig = true
369 }
370
371
372
373 func OptionTransportConfig(tc *TransportConfig) ClientOption {
374 return optionTransportConfig{tc}
375 }
376
377 type optionTransportConfig struct {
378 tc *TransportConfig
379 }
380
381 func (o optionTransportConfig) modifyClient(c *Client) { c.transportConfig = o.tc }
382
383
384
385
386
387 func OptionInsecure(v bool) ClientOption {
388 return optionInsecure(v)
389 }
390
391 type optionInsecure bool
392
393 func (o optionInsecure) modifyClient(c *Client) { c.insecureAnyTLSCert = bool(o) }
394
395
396
397
398
399
400
401 func OptionTrustedCert(cert string) ClientOption {
402
403 return optionTrustedCert(cert)
404 }
405
406 type optionTrustedCert string
407
408 func (o optionTrustedCert) modifyClient(c *Client) {
409 cert := string(o)
410 if cert != "" {
411 c.initTrustedCertsOnce.Do(func() {})
412 c.trustedCerts = []string{string(o)}
413 }
414 }
415
416 type optionNoExtConfig bool
417
418 func (o optionNoExtConfig) modifyClient(c *Client) { c.noExtConfig = bool(o) }
419
420
421
422
423 func OptionNoExternalConfig() ClientOption {
424 return optionNoExtConfig(true)
425 }
426
427 type optionAuthMode struct {
428 m auth.AuthMode
429 }
430
431 func (o optionAuthMode) modifyClient(c *Client) { c.authMode = o.m }
432
433
434
435 func OptionAuthMode(m auth.AuthMode) ClientOption {
436 return optionAuthMode{m}
437 }
438
439
440 func noop() error { return nil }
441
442 var shareURLRx = regexp.MustCompile(`^(.+)/(` + blob.Pattern + ")$")
443
444
445
446 func NewFromShareRoot(ctx context.Context, shareBlobURL string, opts ...ClientOption) (c *Client, target blob.Ref, err error) {
447 var root string
448 m := shareURLRx.FindStringSubmatch(shareBlobURL)
449 if m == nil {
450 return nil, blob.Ref{}, fmt.Errorf("Unknown share URL base")
451 }
452 c, err = New(append(opts[:len(opts):cap(opts)], OptionServer(m[1]))...)
453 if err != nil {
454 return nil, blob.Ref{}, err
455 }
456 c.discoOnce.Do(noop)
457 c.prefixOnce.Do(noop)
458 c.prefixv = m[1]
459 c.isSharePrefix = true
460 c.authMode = auth.None{}
461 c.via = make(map[blob.Ref]blob.Ref)
462 root = m[2]
463
464 req := c.newRequest(ctx, "GET", shareBlobURL, nil)
465 res, err := c.expect2XX(req)
466 if err != nil {
467 return nil, blob.Ref{}, fmt.Errorf("error fetching %s: %v", shareBlobURL, err)
468 }
469 defer res.Body.Close()
470 var buf bytes.Buffer
471 rootbr, ok := blob.Parse(root)
472 if !ok {
473 return nil, blob.Ref{}, fmt.Errorf("invalid root blob ref for sharing: %q", root)
474 }
475 b, err := schema.BlobFromReader(rootbr, io.TeeReader(res.Body, &buf))
476 if err != nil {
477 return nil, blob.Ref{}, fmt.Errorf("error parsing JSON from %s: %v , with response: %q", shareBlobURL, err, buf.Bytes())
478 }
479 if b.ShareAuthType() != schema.ShareHaveRef {
480 return nil, blob.Ref{}, fmt.Errorf("unknown share authType of %q", b.ShareAuthType())
481 }
482 target = b.ShareTarget()
483 if !target.Valid() {
484 return nil, blob.Ref{}, fmt.Errorf("no target")
485 }
486 c.via[target] = rootbr
487 return c, target, nil
488 }
489
490
491
492 func (c *Client) SetHTTPClient(client *http.Client) {
493 if client == nil {
494 client = http.DefaultClient
495 }
496 c.httpClient = client
497 }
498
499
500 func (c *Client) HTTPClient() *http.Client {
501 return c.httpClient
502 }
503
504
505 type HaveCache interface {
506 StatBlobCache(br blob.Ref) (size uint32, ok bool)
507 NoteBlobExists(br blob.Ref, size uint32)
508 }
509
510 type noHaveCache struct{}
511
512 func (noHaveCache) StatBlobCache(blob.Ref) (uint32, bool) { return 0, false }
513 func (noHaveCache) NoteBlobExists(blob.Ref, uint32) {}
514
515 func (c *Client) SetHaveCache(cache HaveCache) {
516 if cache == nil {
517 cache = noHaveCache{}
518 }
519 c.haveCache = cache
520 }
521
522 func (c *Client) printf(format string, v ...interface{}) {
523 if c.Verbose && c.Logger != nil {
524 c.Logger.Printf(format, v...)
525 }
526 }
527
528 func (c *Client) Stats() Stats {
529 c.statsMutex.Lock()
530 defer c.statsMutex.Unlock()
531 return c.stats
532 }
533
534
535 var ErrNoSearchRoot = errors.New("client: server doesn't support search")
536
537
538 var ErrNoHelpRoot = errors.New("client: server does not have a help handler")
539
540
541 var ErrNoShareRoot = errors.New("client: server does not have a share handler")
542
543
544 var ErrNoSigning = fmt.Errorf("client: server doesn't support signing")
545
546
547
548 var ErrNoStorageGeneration = errors.New("client: server doesn't report a storage generation")
549
550
551 var ErrNoSync = errors.New("client: server has no sync handlers")
552
553
554
555
556
557 func (c *Client) BlobRoot() (string, error) {
558 prefix, err := c.prefix()
559 if err != nil {
560 return "", err
561 }
562 return prefix + "/", nil
563 }
564
565
566
567
568 func (c *Client) ServerKeyID() (string, error) {
569 if err := c.condDiscovery(); err != nil {
570 return "", err
571 }
572 if c.serverKeyID == "" {
573 return "", ErrNoSigning
574 }
575 return c.serverKeyID, nil
576 }
577
578
579
580 func (c *Client) ServerPublicKeyBlobRef() (blob.Ref, error) {
581 if err := c.condDiscovery(); err != nil {
582 return blob.Ref{}, err
583 }
584
585 if !c.serverPublicKeyBlobRef.Valid() {
586 return blob.Ref{}, ErrNoSigning
587 }
588 return c.serverPublicKeyBlobRef, nil
589 }
590
591
592
593
594 func (c *Client) SearchRoot() (string, error) {
595 if err := c.condDiscovery(); err != nil {
596 return "", err
597 }
598 if c.searchRoot == "" {
599 return "", ErrNoSearchRoot
600 }
601 return c.searchRoot, nil
602 }
603
604
605
606
607 func (c *Client) HelpRoot() (string, error) {
608 if err := c.condDiscovery(); err != nil {
609 return "", err
610 }
611 if c.helpRoot == "" {
612 return "", ErrNoHelpRoot
613 }
614 return c.helpRoot, nil
615 }
616
617
618
619
620 func (c *Client) ShareRoot() (string, error) {
621 if err := c.condDiscovery(); err != nil {
622 return "", err
623 }
624 if c.shareRoot == "" {
625 return "", ErrNoShareRoot
626 }
627 return c.shareRoot, nil
628 }
629
630
631
632
633 func (c *Client) SignHandler() (string, error) {
634 if err := c.condDiscovery(); err != nil {
635 return "", err
636 }
637 if c.signHandler == "" {
638 return "", ErrNoSigning
639 }
640 return c.signHandler, nil
641 }
642
643
644
645
646
647
648
649
650
651
652 func (c *Client) StorageGeneration() (string, error) {
653 if err := c.condDiscovery(); err != nil {
654 return "", err
655 }
656 if c.storageGen == "" {
657 return "", ErrNoStorageGeneration
658 }
659 return c.storageGen, nil
660 }
661
662
663 func (c *Client) HasLegacySHA1() (bool, error) {
664 if err := c.condDiscovery(); err != nil {
665 return false, err
666 }
667 return c.hasLegacySHA1, nil
668 }
669
670
671
672 type SyncInfo struct {
673 From string
674 To string
675 ToIndex bool
676 }
677
678
679
680
681
682 func (c *Client) SyncHandlers() ([]*SyncInfo, error) {
683 if err := c.condDiscovery(); err != nil {
684 return nil, err
685 }
686 if c.syncHandlers == nil {
687 return nil, ErrNoSync
688 }
689 return c.syncHandlers, nil
690 }
691
692 var _ search.GetRecentPermanoder = (*Client)(nil)
693
694
695 func (c *Client) GetRecentPermanodes(ctx context.Context, req *search.RecentRequest) (*search.RecentResponse, error) {
696 sr, err := c.SearchRoot()
697 if err != nil {
698 return nil, err
699 }
700 url := sr + req.URLSuffix()
701 hreq := c.newRequest(ctx, "GET", url)
702 hres, err := c.expect2XX(hreq)
703 if err != nil {
704 return nil, err
705 }
706 res := new(search.RecentResponse)
707 if err := httputil.DecodeJSON(hres, res); err != nil {
708 return nil, err
709 }
710 if err := res.Err(); err != nil {
711 return nil, err
712 }
713 return res, nil
714 }
715
716
717
718
719
720 func (c *Client) GetPermanodesWithAttr(ctx context.Context, req *search.WithAttrRequest) (*search.WithAttrResponse, error) {
721 sr, err := c.SearchRoot()
722 if err != nil {
723 return nil, err
724 }
725 url := sr + req.URLSuffix()
726 hreq := c.newRequest(ctx, "GET", url)
727 hres, err := c.expect2XX(hreq)
728 if err != nil {
729 return nil, err
730 }
731 res := new(search.WithAttrResponse)
732 if err := httputil.DecodeJSON(hres, res); err != nil {
733 return nil, err
734 }
735 if err := res.Err(); err != nil {
736 return nil, err
737 }
738 return res, nil
739 }
740
741 func (c *Client) Describe(ctx context.Context, req *search.DescribeRequest) (*search.DescribeResponse, error) {
742 sr, err := c.SearchRoot()
743 if err != nil {
744 return nil, err
745 }
746 url := sr + req.URLSuffixPost()
747 body, err := json.MarshalIndent(req, "", "\t")
748 if err != nil {
749 return nil, err
750 }
751 hreq := c.newRequest(ctx, "POST", url, bytes.NewReader(body))
752 hres, err := c.expect2XX(hreq)
753 if err != nil {
754 return nil, err
755 }
756 res := new(search.DescribeResponse)
757 if err := httputil.DecodeJSON(hres, res); err != nil {
758 return nil, err
759 }
760 return res, nil
761 }
762
763 func (c *Client) GetClaims(ctx context.Context, req *search.ClaimsRequest) (*search.ClaimsResponse, error) {
764 sr, err := c.SearchRoot()
765 if err != nil {
766 return nil, err
767 }
768 url := sr + req.URLSuffix()
769 hreq := c.newRequest(ctx, "GET", url)
770 hres, err := c.expect2XX(hreq)
771 if err != nil {
772 return nil, err
773 }
774 res := new(search.ClaimsResponse)
775 if err := httputil.DecodeJSON(hres, res); err != nil {
776 return nil, err
777 }
778 return res, nil
779 }
780
781 func (c *Client) query(ctx context.Context, req *search.SearchQuery) (*http.Response, error) {
782 sr, err := c.SearchRoot()
783 if err != nil {
784 return nil, err
785 }
786 url := sr + req.URLSuffix()
787 body, err := json.Marshal(req)
788 if err != nil {
789 return nil, err
790 }
791 hreq := c.newRequest(ctx, "POST", url, bytes.NewReader(body))
792 return c.expect2XX(hreq)
793 }
794
795 func (c *Client) Query(ctx context.Context, req *search.SearchQuery) (*search.SearchResult, error) {
796 hres, err := c.query(ctx, req)
797 if err != nil {
798 return nil, err
799 }
800 res := new(search.SearchResult)
801 if err := httputil.DecodeJSON(hres, res); err != nil {
802 return nil, err
803 }
804 return res, nil
805 }
806
807
808
809 func (c *Client) QueryRaw(ctx context.Context, req *search.SearchQuery) ([]byte, error) {
810 hres, err := c.query(ctx, req)
811 if err != nil {
812 return nil, err
813 }
814 defer hres.Body.Close()
815 return ioutil.ReadAll(hres.Body)
816 }
817
818
819
820
821
822
823
824
825
826
827
828
829 func (c *Client) SearchExistingFileSchema(ctx context.Context, wholeRef ...blob.Ref) (blob.Ref, error) {
830 sr, err := c.SearchRoot()
831 if err != nil {
832 return blob.Ref{}, err
833 }
834 if len(wholeRef) == 0 {
835 return blob.Ref{}, nil
836 }
837 url := sr + "camli/search/files"
838 for i, ref := range wholeRef {
839 if i == 0 {
840 url += "?wholedigest=" + ref.String()
841 } else {
842 url += "&wholedigest=" + ref.String()
843 }
844 }
845 req := c.newRequest(ctx, "GET", url)
846 res, err := c.doReqGated(req)
847 if err != nil {
848 return blob.Ref{}, err
849 }
850 if res.StatusCode != 200 {
851 body, _ := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20))
852 res.Body.Close()
853 return blob.Ref{}, fmt.Errorf("client: got status code %d from URL %s; body %s", res.StatusCode, url, body)
854 }
855 var ress camtypes.FileSearchResponse
856 if err := httputil.DecodeJSON(res, &ress); err != nil {
857
858 mismatch, err := c.versionMismatch(ctx)
859 if err != nil {
860 log.Printf("Could not verify whether client is too recent or server is too old: %v", err)
861 } else if mismatch {
862 return blob.Ref{}, fmt.Errorf("Client is too recent for this server. Use a client built before 2018-01-13-6e8a5930c9, or upgrade the server to after that revision.")
863 }
864 return blob.Ref{}, fmt.Errorf("client: error parsing JSON from URL %s: %v", url, err)
865 }
866 if len(ress.Files) == 0 {
867 return blob.Ref{}, nil
868 }
869 for wholeRef, files := range ress.Files {
870 for _, f := range files {
871 if c.FileHasContents(ctx, f, blob.MustParse(wholeRef)) {
872 return f, nil
873 }
874 }
875 }
876 return blob.Ref{}, nil
877 }
878
879
880
881 func (c *Client) versionMismatch(ctx context.Context) (bool, error) {
882 const shortRFC3339 = "2006-01-02"
883 version := buildinfo.GitInfo
884 if version == "" {
885 return false, errors.New("unknown client version")
886 }
887 version = version[:10]
888 clientDate, err := time.Parse(shortRFC3339, version)
889 if err != nil {
890 return false, fmt.Errorf("could not parse date from version %q: %v", version, err)
891 }
892 apiChangeDate, _ := time.Parse(shortRFC3339, "2018-01-13")
893 if !clientDate.After(apiChangeDate) {
894
895 return false, nil
896 }
897 url := c.discoRoot() + "/status/status.json"
898 req := c.newRequest(ctx, "GET", url)
899 res, err := c.doReqGated(req)
900 if err != nil {
901 return false, err
902 }
903 if res.StatusCode != 200 {
904 body, _ := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20))
905 res.Body.Close()
906 return false, fmt.Errorf("got status code %d from URL %s; body %s", res.StatusCode, url, body)
907 }
908 var status struct {
909 Version string `json:"version"`
910 }
911 if err := httputil.DecodeJSON(res, &status); err != nil {
912 return false, fmt.Errorf("error parsing JSON from URL %s: %v", url, err)
913 }
914 serverVersion := status.Version[:10]
915 serverDate, err := time.Parse(shortRFC3339, serverVersion)
916 if err != nil {
917 return false, fmt.Errorf("could not parse date from server version %q: %v", status.Version, err)
918 }
919 if serverDate.After(apiChangeDate) {
920
921 return false, nil
922 }
923 return true, nil
924 }
925
926
927
928
929 func (c *Client) FileHasContents(ctx context.Context, f, wholeRef blob.Ref) bool {
930 if err := c.condDiscovery(); err != nil {
931 return false
932 }
933 if c.downloadHelper == "" {
934 return false
935 }
936 req := c.newRequest(ctx, "HEAD", c.downloadHelper+f.String()+"/?verifycontents="+wholeRef.String())
937 res, err := c.expect2XX(req)
938 if err != nil {
939 log.Printf("download helper HEAD error: %v", err)
940 return false
941 }
942 defer res.Body.Close()
943 return res.Header.Get("X-Camli-Contents") == wholeRef.String()
944 }
945
946
947
948
949 func (c *Client) prefix() (string, error) {
950 if err := c.prefixOnce.Do(c.initPrefix); err != nil {
951 return "", err
952 }
953 return c.prefixv, nil
954 }
955
956
957
958 func (c *Client) blobPrefix() (string, error) {
959 pfx, err := c.prefix()
960 if err != nil {
961 return "", err
962 }
963 if !c.isSharePrefix {
964 pfx += "/camli"
965 }
966 return pfx, nil
967 }
968
969
970 func (c *Client) discoRoot() string {
971 s := c.server
972 if c.sameOrigin {
973 s = strings.TrimPrefix(s, "http://")
974 s = strings.TrimPrefix(s, "https://")
975 parts := strings.SplitN(s, "/", 1)
976 if len(parts) < 2 {
977 return "/"
978 }
979 return "/" + parts[1]
980 }
981 if !strings.HasPrefix(s, "http") {
982 s = "https://" + s
983 }
984 return s
985 }
986
987
988
989
990
991 func (c *Client) initPrefix() error {
992 c.isSharePrefix = false
993 root := c.discoRoot()
994 u, err := url.Parse(root)
995 if err != nil {
996 return err
997 }
998 if len(u.Path) > 1 {
999 c.prefixv = strings.TrimRight(root, "/")
1000 return nil
1001 }
1002 return c.condDiscovery()
1003 }
1004
1005 func (c *Client) condDiscovery() error {
1006 if c.sto != nil {
1007 return errors.New("client not using HTTP")
1008 }
1009 return c.discoOnce.Do(c.doDiscovery)
1010 }
1011
1012
1013
1014
1015 func (c *Client) DiscoveryDoc(ctx context.Context) (io.Reader, error) {
1016 res, err := c.discoveryResp(ctx)
1017 if err != nil {
1018 return nil, err
1019 }
1020 defer res.Body.Close()
1021 const maxSize = 1 << 20
1022 all, err := ioutil.ReadAll(io.LimitReader(res.Body, maxSize+1))
1023 if err != nil {
1024 return nil, err
1025 }
1026 if len(all) > maxSize {
1027 return nil, errors.New("discovery document oddly large")
1028 }
1029 if len(all) > 0 && all[len(all)-1] != '\n' {
1030 all = append(all, '\n')
1031 }
1032 return bytes.NewReader(all), err
1033 }
1034
1035
1036 func (c *Client) HTTPVersion(ctx context.Context) (string, error) {
1037 req := c.newRequest(ctx, "HEAD", c.discoRoot(), nil)
1038 res, err := c.doReqGated(req)
1039 if err != nil {
1040 return "", err
1041 }
1042 return res.Proto, err
1043 }
1044
1045 func (c *Client) discoveryResp(ctx context.Context) (*http.Response, error) {
1046
1047
1048 req := c.newRequest(ctx, "GET", c.discoRoot(), nil)
1049 req.Header.Set("Accept", "text/x-camli-configuration")
1050 res, err := c.doReqGated(req)
1051 if err != nil {
1052 return nil, err
1053 }
1054 if res.StatusCode != 200 {
1055 res.Body.Close()
1056 errMsg := fmt.Sprintf("got status %q from blobserver URL %q during configuration discovery", res.Status, c.discoRoot())
1057 if res.StatusCode == 401 && c.authErr != nil {
1058 errMsg = fmt.Sprintf("%v. %v", c.authErr, errMsg)
1059 }
1060 return nil, errors.New(errMsg)
1061 }
1062
1063
1064
1065 if ct := res.Header.Get("Content-Type"); ct != "text/javascript" {
1066 res.Body.Close()
1067 return nil, fmt.Errorf("Blobserver returned unexpected type %q from discovery", ct)
1068 }
1069 return res, nil
1070 }
1071
1072 func (c *Client) doDiscovery() error {
1073 ctx := context.TODO()
1074 root, err := url.Parse(c.discoRoot())
1075 if err != nil {
1076 return err
1077 }
1078
1079 res, err := c.discoveryResp(ctx)
1080 if err != nil {
1081 return err
1082 }
1083
1084 var disco camtypes.Discovery
1085 if err := httputil.DecodeJSON(res, &disco); err != nil {
1086 return err
1087 }
1088
1089 if disco.SearchRoot == "" {
1090 c.searchRoot = ""
1091 } else {
1092 u, err := root.Parse(disco.SearchRoot)
1093 if err != nil {
1094 return fmt.Errorf("client: invalid searchRoot %q; failed to resolve", disco.SearchRoot)
1095 }
1096 c.searchRoot = u.String()
1097 }
1098
1099 u, err := root.Parse(disco.HelpRoot)
1100 if err != nil {
1101 return fmt.Errorf("client: invalid helpRoot %q; failed to resolve", disco.HelpRoot)
1102 }
1103 c.helpRoot = u.String()
1104
1105 u, err = root.Parse(disco.ShareRoot)
1106 if err != nil {
1107 return fmt.Errorf("client: invalid shareRoot %q; failed to resolve", disco.ShareRoot)
1108 }
1109 c.shareRoot = u.String()
1110
1111 c.storageGen = disco.StorageGeneration
1112 c.hasLegacySHA1 = disco.HasLegacySHA1Index
1113
1114 u, err = root.Parse(disco.BlobRoot)
1115 if err != nil {
1116 return fmt.Errorf("client: error resolving blobRoot: %v", err)
1117 }
1118 c.prefixv = strings.TrimRight(u.String(), "/")
1119
1120 if disco.UIDiscovery != nil {
1121 u, err = root.Parse(disco.DownloadHelper)
1122 if err != nil {
1123 return fmt.Errorf("client: invalid downloadHelper %q; failed to resolve", disco.DownloadHelper)
1124 }
1125 c.downloadHelper = u.String()
1126 }
1127
1128 if disco.SyncHandlers != nil {
1129 for _, v := range disco.SyncHandlers {
1130 ufrom, err := root.Parse(v.From)
1131 if err != nil {
1132 return fmt.Errorf("client: invalid %q \"from\" sync; failed to resolve", v.From)
1133 }
1134 uto, err := root.Parse(v.To)
1135 if err != nil {
1136 return fmt.Errorf("client: invalid %q \"to\" sync; failed to resolve", v.To)
1137 }
1138 c.syncHandlers = append(c.syncHandlers, &SyncInfo{
1139 From: ufrom.String(),
1140 To: uto.String(),
1141 ToIndex: v.ToIndex,
1142 })
1143 }
1144 }
1145
1146 if disco.Signing != nil {
1147 c.serverKeyID = disco.Signing.PublicKeyID
1148 c.serverPublicKeyBlobRef = disco.Signing.PublicKeyBlobRef
1149 c.signHandler = disco.Signing.SignHandler
1150 }
1151 return nil
1152 }
1153
1154
1155
1156
1157 func (c *Client) GetJSON(ctx context.Context, url string, data interface{}) error {
1158 if !strings.HasPrefix(url, c.discoRoot()) {
1159 return fmt.Errorf("wrong URL (%q) for this server", url)
1160 }
1161 hreq := c.newRequest(ctx, "GET", url)
1162 resp, err := c.expect2XX(hreq)
1163 if err != nil {
1164 return err
1165 }
1166 return httputil.DecodeJSON(resp, data)
1167 }
1168
1169
1170
1171
1172 func (c *Client) Post(ctx context.Context, url string, bodyType string, body io.Reader) error {
1173 resp, err := c.post(ctx, url, bodyType, body)
1174 if err != nil {
1175 return err
1176 }
1177 return resp.Body.Close()
1178 }
1179
1180
1181
1182
1183 func (c *Client) Sign(ctx context.Context, server string, r io.Reader) (signed []byte, err error) {
1184 signHandler, err := c.SignHandler()
1185 if err != nil {
1186 return nil, err
1187 }
1188 signServer := strings.TrimSuffix(server, "/") + signHandler
1189 resp, err := c.post(ctx, signServer, "application/x-www-form-urlencoded", r)
1190 if err != nil {
1191 return nil, err
1192 }
1193 defer resp.Body.Close()
1194 return ioutil.ReadAll(resp.Body)
1195 }
1196
1197 func (c *Client) post(ctx context.Context, url string, bodyType string, body io.Reader) (*http.Response, error) {
1198 if !c.sameOrigin && !strings.HasPrefix(url, c.discoRoot()) {
1199 return nil, fmt.Errorf("wrong URL (%q) for this server", url)
1200 }
1201 req := c.newRequest(ctx, "POST", url, body)
1202 req.Header.Set("Content-Type", bodyType)
1203 res, err := c.expect2XX(req)
1204 if err != nil {
1205 return nil, err
1206 }
1207 return res, nil
1208 }
1209
1210
1211
1212 func (c *Client) newRequest(ctx context.Context, method, url string, body ...io.Reader) *http.Request {
1213 var bodyR io.Reader
1214 if len(body) > 0 {
1215 bodyR = body[0]
1216 }
1217 if len(body) > 1 {
1218 panic("too many body arguments")
1219 }
1220 req, err := http.NewRequest(method, url, bodyR)
1221 if err != nil {
1222 panic(err.Error())
1223 }
1224
1225 if br, ok := bodyR.(*bytes.Reader); ok {
1226 req.ContentLength = int64(br.Len())
1227 }
1228 c.authMode.AddAuthHeader(req)
1229 return req.WithContext(ctx)
1230 }
1231
1232
1233
1234 func (c *Client) expect2XX(req *http.Request) (*http.Response, error) {
1235 res, err := c.doReqGated(req)
1236 if err == nil && (res.StatusCode < 200 || res.StatusCode > 299) {
1237 buf := new(bytes.Buffer)
1238 io.CopyN(buf, res.Body, 1<<20)
1239 res.Body.Close()
1240 return res, fmt.Errorf("client: got status code %d from URL %s; body %s", res.StatusCode, req.URL.String(), buf.String())
1241 }
1242 return res, err
1243 }
1244
1245 func (c *Client) doReqGated(req *http.Request) (*http.Response, error) {
1246 c.httpGate.Start()
1247 defer c.httpGate.Done()
1248 return c.httpClient.Do(req)
1249 }
1250
1251
1252 func (c *Client) DialFunc() func(network, addr string) (net.Conn, error) {
1253 if c.useTLS() {
1254 return nil
1255 }
1256 if android.IsChild() {
1257 return func(network, addr string) (net.Conn, error) {
1258 return android.Dial(network, addr)
1259 }
1260 }
1261 return nil
1262 }
1263
1264 func (c *Client) http2DialTLSFunc() func(network, addr string, cfg *tls.Config) (net.Conn, error) {
1265 trustedCerts := c.getTrustedCerts()
1266 if !c.insecureAnyTLSCert && len(trustedCerts) == 0 {
1267
1268
1269 return nil
1270 }
1271 return func(network, addr string, cfg *tls.Config) (net.Conn, error) {
1272
1273 cfg.InsecureSkipVerify = true
1274 conn, err := tls.Dial(network, addr, cfg)
1275 if err != nil {
1276 return nil, err
1277 }
1278 if c.insecureAnyTLSCert {
1279 return conn, err
1280 }
1281 state := conn.ConnectionState()
1282 if p := state.NegotiatedProtocol; p != http2.NextProtoTLS {
1283 return nil, fmt.Errorf("http2: unexpected ALPN protocol %q; want %q", p, http2.NextProtoTLS)
1284 }
1285 if !state.NegotiatedProtocolIsMutual {
1286 return nil, errors.New("http2: could not negotiate protocol mutually")
1287 }
1288 certs := state.PeerCertificates
1289 if len(certs) < 1 {
1290 return nil, fmt.Errorf("no TLS peer certificates from %s", addr)
1291 }
1292 sig := hashutil.SHA256Prefix(certs[0].Raw)
1293 for _, v := range trustedCerts {
1294 if v == sig {
1295 return conn, nil
1296 }
1297 }
1298 return nil, fmt.Errorf("TLS server at %v presented untrusted certificate (signature %q)", addr, sig)
1299 }
1300 }
1301
1302
1303
1304
1305
1306
1307 func (c *Client) DialTLSFunc() func(network, addr string) (net.Conn, error) {
1308 if !c.useTLS() {
1309 return nil
1310 }
1311 trustedCerts := c.getTrustedCerts()
1312 var stdTLS bool
1313 if !c.insecureAnyTLSCert && len(trustedCerts) == 0 {
1314
1315 stdTLS = true
1316 if !android.IsChild() {
1317
1318 return nil
1319 }
1320 }
1321
1322 return func(network, addr string) (net.Conn, error) {
1323 var conn *tls.Conn
1324 var err error
1325 if android.IsChild() {
1326 ac, err := android.Dial(network, addr)
1327 if err != nil {
1328 return nil, err
1329 }
1330 var tlsConfig *tls.Config
1331 if stdTLS {
1332 tlsConfig, err = android.TLSConfig()
1333 if err != nil {
1334 return nil, err
1335 }
1336 } else {
1337 tlsConfig = &tls.Config{InsecureSkipVerify: true}
1338 }
1339
1340
1341
1342 tlsConfig.ServerName = c.serverNameOfAddr(addr)
1343 conn = tls.Client(ac, tlsConfig)
1344 if err := conn.Handshake(); err != nil {
1345 return nil, err
1346 }
1347 if stdTLS {
1348
1349
1350 return conn, nil
1351 }
1352 } else {
1353 conn, err = tls.Dial(network, addr, &tls.Config{InsecureSkipVerify: true})
1354 if err != nil {
1355 return nil, err
1356 }
1357 }
1358 if c.insecureAnyTLSCert {
1359 return conn, nil
1360 }
1361 certs := conn.ConnectionState().PeerCertificates
1362 if len(certs) < 1 {
1363 return nil, fmt.Errorf("no TLS peer certificates from %s", addr)
1364 }
1365 sig := hashutil.SHA256Prefix(certs[0].Raw)
1366 for _, v := range trustedCerts {
1367 if v == sig {
1368 return conn, nil
1369 }
1370 }
1371 return nil, fmt.Errorf("TLS server at %v presented untrusted certificate (signature %q)", addr, sig)
1372 }
1373 }
1374
1375
1376
1377
1378 func (c *Client) serverNameOfAddr(addr string) string {
1379 serverName, _, err := net.SplitHostPort(addr)
1380 if err != nil {
1381 c.printf("could not get server name from address %q: %v", addr, err)
1382 return ""
1383 }
1384 if ip := net.ParseIP(serverName); ip != nil {
1385 return ""
1386 }
1387 return serverName
1388 }
1389
1390
1391
1392 func (c *Client) Signer() (*schema.Signer, error) {
1393 c.signerOnce.Do(c.signerInit)
1394 return c.signer, c.signerErr
1395 }
1396
1397 func (c *Client) signerInit() {
1398 c.signer, c.signerErr = c.buildSigner()
1399 }
1400
1401 func (c *Client) buildSigner() (*schema.Signer, error) {
1402 c.initSignerPublicKeyBlobrefOnce.Do(c.initSignerPublicKeyBlobref)
1403 if !c.signerPublicKeyRef.Valid() {
1404 return nil, camtypes.ErrClientNoPublicKey
1405 }
1406 return schema.NewSigner(c.signerPublicKeyRef, strings.NewReader(c.publicKeyArmored), c.SecretRingFile())
1407 }
1408
1409
1410
1411 func (c *Client) signBlob(ctx context.Context, bb schema.Buildable, sigTime time.Time) (string, error) {
1412 signer, err := c.Signer()
1413 if err != nil {
1414 return "", err
1415 }
1416 return bb.Builder().SignAt(ctx, signer, sigTime)
1417 }
1418
1419
1420
1421
1422
1423 func (c *Client) uploadPublicKey(ctx context.Context) error {
1424 sigRef := c.SignerPublicKeyBlobref()
1425 if !sigRef.Valid() {
1426 return nil
1427 }
1428 var err error
1429 if _, keyUploaded := c.haveCache.StatBlobCache(sigRef); !keyUploaded {
1430 _, err = c.uploadString(ctx, c.publicKeyArmored, false)
1431 }
1432 return err
1433 }
1434
1435
1436 func (c *Client) checkMatchingKeys() {
1437 serverKey, err := c.ServerKeyID()
1438 if err != nil {
1439 log.Printf("Warning: Could not obtain ther server's key id: %v", err)
1440 return
1441 }
1442 if serverKey != c.signer.KeyIDLong() {
1443 log.Printf("Warning: client (%s) and server (%s) keys differ.", c.signer.KeyIDLong(), serverKey)
1444 }
1445 }
1446
1447 func (c *Client) UploadAndSignBlob(ctx context.Context, b schema.AnyBlob) (*PutResult, error) {
1448 signed, err := c.signBlob(ctx, b.Blob(), time.Time{})
1449 if err != nil {
1450 return nil, err
1451 }
1452 c.checkMatchingKeys()
1453 if err := c.uploadPublicKey(ctx); err != nil {
1454 return nil, err
1455 }
1456 return c.uploadString(ctx, signed, false)
1457 }
1458
1459 func (c *Client) UploadBlob(ctx context.Context, b schema.AnyBlob) (*PutResult, error) {
1460
1461
1462 return c.uploadString(ctx, b.Blob().JSON(), true)
1463 }
1464
1465 func (c *Client) uploadString(ctx context.Context, s string, stat bool) (*PutResult, error) {
1466 uh := NewUploadHandleFromString(s)
1467 uh.SkipStat = !stat
1468 return c.Upload(ctx, uh)
1469 }
1470
1471 func (c *Client) UploadNewPermanode(ctx context.Context) (*PutResult, error) {
1472 unsigned := schema.NewUnsignedPermanode()
1473 return c.UploadAndSignBlob(ctx, unsigned)
1474 }
1475
1476 func (c *Client) UploadPlannedPermanode(ctx context.Context, key string, sigTime time.Time) (*PutResult, error) {
1477 unsigned := schema.NewPlannedPermanode(key)
1478 signed, err := c.signBlob(ctx, unsigned, sigTime)
1479 if err != nil {
1480 return nil, err
1481 }
1482 c.checkMatchingKeys()
1483 if err := c.uploadPublicKey(ctx); err != nil {
1484 return nil, err
1485 }
1486 return c.uploadString(ctx, signed, true)
1487 }
1488
1489
1490
1491
1492
1493
1494
1495 func (c *Client) IsIgnoredFile(fullpath string) bool {
1496 c.initIgnoredFilesOnce.Do(c.initIgnoredFiles)
1497 return c.ignoreChecker(fullpath)
1498 }
1499
1500
1501 func (c *Client) Close() error {
1502 if cl, ok := c.sto.(io.Closer); ok {
1503 return cl.Close()
1504 }
1505 if c := c.HTTPClient(); c != nil {
1506 switch t := c.Transport.(type) {
1507 case *http.Transport:
1508 t.CloseIdleConnections()
1509 case *http2.Transport:
1510 t.CloseIdleConnections()
1511 }
1512 }
1513 return nil
1514 }
1515
1516 func httpGateSize(rt http.RoundTripper) int {
1517 switch v := rt.(type) {
1518 case *httputil.StatsTransport:
1519 return httpGateSize(v.Transport)
1520 case *http.Transport:
1521 return maxParallelHTTP_h1
1522 case *http2.Transport:
1523 return maxParallelHTTP_h2
1524 default:
1525 return maxParallelHTTP_h1
1526 }
1527 }