1	/*
     2	Copyright 2014 The Perkeep Authors
     3	
     4	Licensed under the Apache License, Version 2.0 (the "License");
     5	you may not use this file except in compliance with the License.
     6	You may obtain a copy of the License at
     7	
     8	     http://www.apache.org/licenses/LICENSE-2.0
     9	
    10	Unless required by applicable law or agreed to in writing, software
    11	distributed under the License is distributed on an "AS IS" BASIS,
    12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13	See the License for the specific language governing permissions and
    14	limitations under the License.
    15	*/
    16	
    17	// Package swarm implements an importer for Foursquare Swarm check-ins.
    18	package swarm // import "perkeep.org/pkg/importer/swarm"
    19	
    20	import (
    21		"context"
    22		"fmt"
    23		"log"
    24		"net/http"
    25		"net/url"
    26		"path"
    27		"path/filepath"
    28		"sort"
    29		"strconv"
    30		"strings"
    31		"sync"
    32		"time"
    33	
    34		"perkeep.org/internal/httputil"
    35		"perkeep.org/pkg/blob"
    36		"perkeep.org/pkg/importer"
    37		"perkeep.org/pkg/schema"
    38		"perkeep.org/pkg/schema/nodeattr"
    39	
    40		"go4.org/ctxutil"
    41		"golang.org/x/oauth2"
    42	)
    43	
    44	const (
    45		apiURL   = "https://api.foursquare.com/v2/"
    46		authURL  = "https://foursquare.com/oauth2/authenticate"
    47		tokenURL = "https://foursquare.com/oauth2/access_token"
    48	
    49		apiVersion      = "20140225"
    50		checkinsAPIPath = "users/self/checkins"
    51	
    52		// runCompleteVersion is a cache-busting version number of the
    53		// importer code. It should be incremented whenever the
    54		// behavior of this importer is updated enough to warrant a
    55		// complete run.  Otherwise, if the importer runs to
    56		// completion, this version number is recorded on the account
    57		// permanode and subsequent importers can stop early.
    58		runCompleteVersion = "2"
    59	
    60		// Permanode attributes on account node:
    61		acctAttrUserId      = "foursquareUserId"
    62		acctAttrUserFirst   = "foursquareFirstName"
    63		acctAttrUserLast    = "foursquareLastName"
    64		acctAttrAccessToken = "oauthAccessToken"
    65	
    66		checkinsRequestLimit = 100 // max number of checkins we will ask for in a checkins list request
    67		photosRequestLimit   = 5
    68	
    69		attrFoursquareId             = "foursquareId"
    70		attrFoursquareVenuePermanode = "foursquareVenuePermanode"
    71		attrFoursquareCategoryName   = "foursquareCategoryName"
    72	)
    73	
    74	func init() {
    75		importer.Register("swarm", &imp{
    76			imageFileRef: make(map[string]blob.Ref),
    77		})
    78	}
    79	
    80	var _ importer.ImporterSetupHTMLer = (*imp)(nil)
    81	
    82	type imp struct {
    83		mu           sync.Mutex          // guards following
    84		imageFileRef map[string]blob.Ref // url to file schema blob
    85	
    86		importer.OAuth2 // for CallbackRequestAccount and CallbackURLParameters
    87	}
    88	
    89	func (*imp) Properties() importer.Properties {
    90		return importer.Properties{
    91			Title:                 "Swarm",
    92			Description:           "import check-ins and venues from Foursquare Swarm (swarmapp.com)",
    93			SupportsIncremental:   true,
    94			NeedsAPIKey:           true,
    95			PermanodeImporterType: "foursquare", // old brand name
    96		}
    97	}
    98	
    99	func (im *imp) IsAccountReady(acctNode *importer.Object) (ok bool, err error) {
   100		if acctNode.Attr(acctAttrUserId) != "" && acctNode.Attr(acctAttrAccessToken) != "" {
   101			return true, nil
   102		}
   103		return false, nil
   104	}
   105	
   106	func (im *imp) SummarizeAccount(acct *importer.Object) string {
   107		ok, err := im.IsAccountReady(acct)
   108		if err != nil {
   109			return "Not configured; error = " + err.Error()
   110		}
   111		if !ok {
   112			return "Not configured"
   113		}
   114		if acct.Attr(acctAttrUserFirst) == "" && acct.Attr(acctAttrUserLast) == "" {
   115			return fmt.Sprintf("userid %s", acct.Attr(acctAttrUserId))
   116		}
   117		return fmt.Sprintf("userid %s (%s %s)", acct.Attr(acctAttrUserId),
   118			acct.Attr(acctAttrUserFirst), acct.Attr(acctAttrUserLast))
   119	}
   120	
   121	func (im *imp) AccountSetupHTML(host *importer.Host) string {
   122		base := host.ImporterBaseURL() + "swarm"
   123		return fmt.Sprintf(`
   124	<h1>Configuring Foursquare</h1>
   125	<p>Visit <a href='https://foursquare.com/developers/apps'>https://foursquare.com/developers/apps</a> and click "Create a new app".</p>
   126	<p>Use the following settings:</p>
   127	<ul>
   128	  <li>Download / welcome page url: <b>%s</b></li>
   129	  <li>Your privacy policy url: <b>%s</b></li>
   130	  <li>Redirect URI(s): <b>%s</b></li>
   131	</ul>
   132	<p>Click "SAVE CHANGES".  Copy the "Client ID" and "Client Secret" into the boxes above.</p>
   133	`, base, base+"/privacy", base+"/callback")
   134	}
   135	
   136	// A run is our state for a given run of the importer.
   137	type run struct {
   138		*importer.RunContext
   139		im          *imp
   140		incremental bool // whether we've completed a run in the past
   141	
   142		mu     sync.Mutex // guards anyErr
   143		anyErr bool
   144	}
   145	
   146	func (r *run) token() string {
   147		return r.RunContext.AccountNode().Attr(acctAttrAccessToken)
   148	}
   149	
   150	func (im *imp) Run(ctx *importer.RunContext) error {
   151		r := &run{
   152			RunContext:  ctx,
   153			im:          im,
   154			incremental: ctx.AccountNode().Attr(importer.AcctAttrCompletedVersion) == runCompleteVersion,
   155		}
   156	
   157		if err := r.importCheckins(); err != nil {
   158			return err
   159		}
   160	
   161		r.mu.Lock()
   162		anyErr := r.anyErr
   163		r.mu.Unlock()
   164	
   165		if !anyErr {
   166			if err := r.AccountNode().SetAttrs(importer.AcctAttrCompletedVersion, runCompleteVersion); err != nil {
   167				return err
   168			}
   169		}
   170	
   171		return nil
   172	}
   173	
   174	func (r *run) errorf(format string, args ...interface{}) {
   175		log.Printf(format, args...)
   176		r.mu.Lock()
   177		defer r.mu.Unlock()
   178		r.anyErr = true
   179	}
   180	
   181	// urlFileRef slurps urlstr from the net, writes to a file and returns its
   182	// fileref or "" on error or if urlstr was empty.
   183	func (r *run) urlFileRef(urlstr, filename string) string {
   184		im := r.im
   185		im.mu.Lock()
   186		if br, ok := im.imageFileRef[urlstr]; ok {
   187			im.mu.Unlock()
   188			return br.String()
   189		}
   190		im.mu.Unlock()
   191	
   192		if urlstr == "" {
   193			return ""
   194		}
   195		res, err := ctxutil.Client(r.Context()).Get(urlstr)
   196		if err != nil {
   197			log.Printf("swarm: couldn't fetch image %q: %v", urlstr, err)
   198			return ""
   199		}
   200		defer res.Body.Close()
   201	
   202		fileRef, err := schema.WriteFileFromReader(r.Context(), r.Host.Target(), filename, res.Body)
   203		if err != nil {
   204			r.errorf("couldn't write file: %v", err)
   205			return ""
   206		}
   207	
   208		im.mu.Lock()
   209		defer im.mu.Unlock()
   210		im.imageFileRef[urlstr] = fileRef
   211		return fileRef.String()
   212	}
   213	
   214	type byCreatedAt []*checkinItem
   215	
   216	func (s byCreatedAt) Less(i, j int) bool {
   217		return s[i].CreatedAt < s[j].CreatedAt
   218	}
   219	func (s byCreatedAt) Len() int {
   220		return len(s)
   221	}
   222	func (s byCreatedAt) Swap(i, j int) {
   223		s[i], s[j] = s[j], s[i]
   224	}
   225	
   226	func (r *run) importCheckins() error {
   227		limit := checkinsRequestLimit
   228		offset := 0
   229		continueRequests := true
   230	
   231		for continueRequests {
   232			resp := checkinsList{}
   233			if err := r.im.doUserAPI(r.Context(), r.token(), &resp, checkinsAPIPath, "limit", strconv.Itoa(limit), "offset", strconv.Itoa(offset)); err != nil {
   234				return err
   235			}
   236	
   237			itemcount := len(resp.Response.Checkins.Items)
   238			log.Printf("swarm: importing %d checkins (offset %d)", itemcount, offset)
   239			if itemcount < limit {
   240				continueRequests = false
   241			} else {
   242				offset += itemcount
   243			}
   244	
   245			checkinsNode, err := r.getTopLevelNode("checkins", "Checkins")
   246			if err != nil {
   247				return err
   248			}
   249	
   250			placesNode, err := r.getTopLevelNode("places", "Places")
   251			if err != nil {
   252				return err
   253			}
   254	
   255			pplNode, err := r.getTopLevelNode("people", "People")
   256			if err != nil {
   257				return err
   258			}
   259	
   260			sort.Sort(byCreatedAt(resp.Response.Checkins.Items))
   261			sawOldItem := false
   262			for _, checkin := range resp.Response.Checkins.Items {
   263				placeNode, err := r.importPlace(placesNode, &checkin.Venue)
   264				if err != nil {
   265					r.errorf("Foursquare importer: error importing place %s: %v", checkin.Venue.Id, err)
   266					continue
   267				}
   268	
   269				companionRefs, err := r.importCompanions(pplNode, checkin.With)
   270				if err != nil {
   271					r.errorf("Foursquare importer: error importing companions for checkin %s: %v", checkin.Id, err)
   272					continue
   273				}
   274	
   275				_, dup, err := r.importCheckin(checkinsNode, checkin, placeNode.PermanodeRef(), companionRefs)
   276				if err != nil {
   277					r.errorf("Foursquare importer: error importing checkin %s: %v", checkin.Id, err)
   278					continue
   279				}
   280	
   281				if dup {
   282					sawOldItem = true
   283				}
   284	
   285				err = r.importPhotos(placeNode, dup)
   286				if err != nil {
   287					r.errorf("Foursquare importer: error importing photos for checkin %s: %v", checkin.Id, err)
   288					continue
   289				}
   290			}
   291			if sawOldItem && r.incremental {
   292				break
   293			}
   294		}
   295	
   296		return nil
   297	}
   298	
   299	func (r *run) importPhotos(placeNode *importer.Object, checkinWasDup bool) error {
   300		photosNode, err := placeNode.ChildPathObject("photos")
   301		if err != nil {
   302			return err
   303		}
   304	
   305		if err := photosNode.SetAttrs(
   306			nodeattr.Title, "Photos of "+placeNode.Attr("title"),
   307			nodeattr.DefaultVisibility, "hide"); err != nil {
   308			return err
   309		}
   310	
   311		nHave := 0
   312		photosNode.ForeachAttr(func(key, value string) {
   313			if strings.HasPrefix(key, "camliPath:") {
   314				nHave++
   315			}
   316		})
   317		nWant := photosRequestLimit
   318		if checkinWasDup {
   319			nWant = 1
   320		}
   321		if nHave >= nWant {
   322			return nil
   323		}
   324	
   325		clientID, clientSecret, err := r.Credentials()
   326		if err != nil {
   327			return err
   328		}
   329	
   330		resp := photosList{}
   331		if err = r.im.doCredAPI(r.Context(), clientID, clientSecret, &resp,
   332			"venues/"+placeNode.Attr(attrFoursquareId)+"/photos",
   333			"limit", strconv.Itoa(nWant)); err != nil {
   334			return err
   335		}
   336	
   337		var need []*photoItem
   338		for _, photo := range resp.Response.Photos.Items {
   339			attr := "camliPath:" + photo.Id + filepath.Ext(photo.Suffix)
   340			if photosNode.Attr(attr) == "" {
   341				need = append(need, photo)
   342			}
   343		}
   344	
   345		if len(need) > 0 {
   346			venueTitle := placeNode.Attr(nodeattr.Title)
   347			log.Printf("swarm: importing %d photos for venue %s", len(need), venueTitle)
   348			for _, photo := range need {
   349				attr := "camliPath:" + photo.Id + filepath.Ext(photo.Suffix)
   350				if photosNode.Attr(attr) != "" {
   351					continue
   352				}
   353				url := photo.Prefix + "original" + photo.Suffix
   354				log.Printf("swarm: importing photo for venue %s: %s", venueTitle, url)
   355				ref := r.urlFileRef(url, "")
   356				if ref == "" {
   357					r.errorf("Error slurping photo: %s", url)
   358					continue
   359				}
   360				if err := photosNode.SetAttr(attr, ref); err != nil {
   361					r.errorf("Error adding venue photo: %#v", err)
   362				}
   363			}
   364		}
   365	
   366		return nil
   367	}
   368	
   369	func (r *run) importCheckin(parent *importer.Object, checkin *checkinItem, placeRef blob.Ref, companionRefs []string) (checkinNode *importer.Object, dup bool, err error) {
   370		checkinNode, err = parent.ChildPathObject(checkin.Id)
   371		if err != nil {
   372			return
   373		}
   374	
   375		title := fmt.Sprintf("Checkin at %s", checkin.Venue.Name)
   376		dup = checkinNode.Attr(nodeattr.StartDate) != ""
   377		if err := checkinNode.SetAttrs(
   378			attrFoursquareId, checkin.Id,
   379			attrFoursquareVenuePermanode, placeRef.String(),
   380			nodeattr.Type, "foursquare.com:checkin",
   381			nodeattr.StartDate, schema.RFC3339FromTime(time.Unix(checkin.CreatedAt, 0)),
   382			nodeattr.Title, title); err != nil {
   383			return nil, false, err
   384		}
   385	
   386		if err := checkinNode.SetAttrValues("with", companionRefs); err != nil {
   387			return nil, false, err
   388		}
   389	
   390		return checkinNode, dup, nil
   391	}
   392	
   393	func (r *run) importCompanions(parent *importer.Object, companions []*user) (companionRefs []string, err error) {
   394		for _, user := range companions {
   395			personNode, err := parent.ChildPathObject(user.Id)
   396			if err != nil {
   397				return nil, err
   398			}
   399			attrs := []string{
   400				attrFoursquareId, user.Id,
   401				nodeattr.Type, "foursquare.com:person",
   402				nodeattr.Title, user.FirstName + " " + user.LastName,
   403				nodeattr.GivenName, user.FirstName,
   404				nodeattr.FamilyName, user.LastName,
   405			}
   406			if icon := user.icon(); icon != "" {
   407				attrs = append(attrs, nodeattr.CamliContentImage, r.urlFileRef(icon, path.Base(icon)))
   408			}
   409			if err := personNode.SetAttrs(attrs...); err != nil {
   410				return nil, err
   411			}
   412			companionRefs = append(companionRefs, personNode.PermanodeRef().String())
   413		}
   414		return companionRefs, nil
   415	}
   416	
   417	func (r *run) importPlace(parent *importer.Object, place *venueItem) (*importer.Object, error) {
   418		placeNode, err := parent.ChildPathObject(place.Id)
   419		if err != nil {
   420			return nil, err
   421		}
   422	
   423		catName := ""
   424		if cat := place.primaryCategory(); cat != nil {
   425			catName = cat.Name
   426		}
   427	
   428		attrs := []string{
   429			attrFoursquareId, place.Id,
   430			nodeattr.Type, "foursquare.com:venue",
   431			attrFoursquareCategoryName, catName,
   432			nodeattr.Title, place.Name,
   433		}
   434		if icon := place.icon(); icon != "" {
   435			attrs = append(attrs,
   436				nodeattr.CamliContentImage, r.urlFileRef(icon, path.Base(icon)))
   437		}
   438		if place.Location != nil {
   439			attrs = append(attrs,
   440				nodeattr.StreetAddress, place.Location.Address,
   441				nodeattr.AddressLocality, place.Location.City,
   442				nodeattr.PostalCode, place.Location.PostalCode,
   443				nodeattr.AddressRegion, place.Location.State,
   444				nodeattr.AddressCountry, place.Location.Country,
   445				nodeattr.Latitude, fmt.Sprint(place.Location.Lat),
   446				nodeattr.Longitude, fmt.Sprint(place.Location.Lng))
   447		}
   448		if err := placeNode.SetAttrs(attrs...); err != nil {
   449			return nil, err
   450		}
   451	
   452		return placeNode, nil
   453	}
   454	
   455	func (r *run) getTopLevelNode(path string, title string) (*importer.Object, error) {
   456		childObject, err := r.RootNode().ChildPathObject(path)
   457		if err != nil {
   458			return nil, err
   459		}
   460	
   461		if err := childObject.SetAttr(nodeattr.Title, title); err != nil {
   462			return nil, err
   463		}
   464		return childObject, nil
   465	}
   466	
   467	func (im *imp) getUserInfo(ctx context.Context, accessToken string) (user, error) {
   468		var ui userInfo
   469		if err := im.doUserAPI(ctx, accessToken, &ui, "users/self"); err != nil {
   470			return user{}, err
   471		}
   472		if ui.Response.User.Id == "" {
   473			return user{}, fmt.Errorf("No userid returned")
   474		}
   475		return ui.Response.User, nil
   476	}
   477	
   478	// doUserAPI makes requests to the Foursquare API with a user token.
   479	// https://developer.foursquare.com/overview/auth#requests
   480	func (im *imp) doUserAPI(ctx context.Context, accessToken string, result interface{}, apiPath string, keyval ...string) error {
   481		form := url.Values{}
   482		form.Set("oauth_token", accessToken)
   483		return im.doAPI(ctx, form, result, apiPath, keyval...)
   484	}
   485	
   486	// doCredAPI makes userless requests to the Foursquare API, which have a larger
   487	// quota than user requests for some endpoints.
   488	// https://developer.foursquare.com/overview/auth#userless
   489	// https://developer.foursquare.com/overview/ratelimits
   490	func (im *imp) doCredAPI(ctx context.Context, clientID, clientSecret string, result interface{}, apiPath string, keyval ...string) error {
   491		form := url.Values{}
   492		form.Set("client_id", clientID)
   493		form.Set("client_secret", clientSecret)
   494		return im.doAPI(ctx, form, result, apiPath, keyval...)
   495	}
   496	
   497	func (im *imp) doAPI(ctx context.Context, form url.Values, result interface{}, apiPath string, keyval ...string) error {
   498		if len(keyval)%2 == 1 {
   499			panic("Incorrect number of keyval arguments")
   500		}
   501	
   502		form.Set("v", apiVersion) // 4sq requires this to version their API
   503		for i := 0; i < len(keyval); i += 2 {
   504			form.Set(keyval[i], keyval[i+1])
   505		}
   506	
   507		fullURL := apiURL + apiPath
   508		res, err := doGet(ctx, fullURL, form)
   509		if err != nil {
   510			return err
   511		}
   512		err = httputil.DecodeJSON(res, result)
   513		if err != nil {
   514			log.Printf("Error parsing response for %s: %v", fullURL, err)
   515		}
   516		return err
   517	}
   518	
   519	func doGet(ctx context.Context, url string, form url.Values) (*http.Response, error) {
   520		requestURL := url + "?" + form.Encode()
   521		req, err := http.NewRequest("GET", requestURL, nil)
   522		if err != nil {
   523			return nil, err
   524		}
   525		res, err := ctxutil.Client(ctx).Do(req)
   526		if err != nil {
   527			log.Printf("Error fetching %s: %v", url, err)
   528			return nil, err
   529		}
   530		if res.StatusCode != http.StatusOK {
   531			return nil, fmt.Errorf("Get request on %s failed with: %s", requestURL, res.Status)
   532		}
   533		return res, nil
   534	}
   535	
   536	// auth returns a new oauth2 Config
   537	func auth(ctx *importer.SetupContext) (*oauth2.Config, error) {
   538		clientID, secret, err := ctx.Credentials()
   539		if err != nil {
   540			return nil, err
   541		}
   542		return &oauth2.Config{
   543			ClientID:     clientID,
   544			ClientSecret: secret,
   545			Endpoint: oauth2.Endpoint{
   546				AuthURL:  authURL,
   547				TokenURL: tokenURL,
   548			},
   549			RedirectURL: ctx.CallbackURL(),
   550			// No scope needed for foursquare as far as I can tell
   551		}, nil
   552	}
   553	
   554	func (im *imp) ServeSetup(w http.ResponseWriter, r *http.Request, ctx *importer.SetupContext) error {
   555		oauthConfig, err := auth(ctx)
   556		if err != nil {
   557			return err
   558		}
   559		oauthConfig.RedirectURL = im.RedirectURL(im, ctx)
   560		state, err := im.RedirectState(im, ctx)
   561		if err != nil {
   562			return err
   563		}
   564		http.Redirect(w, r, oauthConfig.AuthCodeURL(state), http.StatusFound)
   565		return nil
   566	}
   567	
   568	func (im *imp) ServeCallback(w http.ResponseWriter, r *http.Request, ctx *importer.SetupContext) {
   569		oauthConfig, err := auth(ctx)
   570		if err != nil {
   571			httputil.ServeError(w, r, fmt.Errorf("Error getting oauth config: %v", err))
   572			return
   573		}
   574	
   575		if r.Method != "GET" {
   576			http.Error(w, "Expected a GET", http.StatusBadRequest)
   577			return
   578		}
   579		code := r.FormValue("code")
   580		if code == "" {
   581			http.Error(w, "Expected a code", http.StatusBadRequest)
   582			return
   583		}
   584		token, err := oauthConfig.Exchange(ctx, code)
   585		log.Printf("Token = %#v, error %v", token, err)
   586		if err != nil {
   587			log.Printf("Token Exchange error: %v", err)
   588			http.Error(w, "token exchange error", 500)
   589			return
   590		}
   591	
   592		u, err := im.getUserInfo(ctx.Context, token.AccessToken)
   593		if err != nil {
   594			log.Printf("Couldn't get username: %v", err)
   595			http.Error(w, "can't get username", 500)
   596			return
   597		}
   598		if err := ctx.AccountNode.SetAttrs(
   599			acctAttrUserId, u.Id,
   600			acctAttrUserFirst, u.FirstName,
   601			acctAttrUserLast, u.LastName,
   602			acctAttrAccessToken, token.AccessToken,
   603		); err != nil {
   604			httputil.ServeError(w, r, fmt.Errorf("Error setting attribute: %v", err))
   605			return
   606		}
   607		http.Redirect(w, r, ctx.AccountURL(), http.StatusFound)
   608	
   609	}