From dccfae7a6137afbdb89c45026fdd1932d073c45e Mon Sep 17 00:00:00 2001 From: Michael Demetriou Date: Tue, 8 Oct 2019 15:58:19 +0300 Subject: [PATCH 01/19] Mentioning pleroma accounts works! Mastodon still needs the type to b be Note to work but I will open an issue for them and see what their reaction will be. --- activitypub.go | 25 ++++++++++++++++++++++++- go.mod | 5 +++++ go.sum | 7 +++++++ posts.go | 18 ++++++++++++++++++ webfinger.go | 31 ++++++++++++++++++++++++++++++- 5 files changed, 84 insertions(+), 2 deletions(-) diff --git a/activitypub.go b/activitypub.go index e37fb97..68ac7c9 100644 --- a/activitypub.go +++ b/activitypub.go @@ -26,6 +26,7 @@ import ( "github.com/gorilla/mux" "github.com/writeas/activity/streams" + "github.com/writeas/activityserve" "github.com/writeas/httpsig" "github.com/writeas/impart" "github.com/writeas/nerds/store" @@ -594,12 +595,12 @@ func federatePost(app *App, p *PublicPost, collID int64, isUpdate bool) error { } } + var activity *activitystreams.Activity for si, instFolls := range inboxes { na.CC = []string{} for _, f := range instFolls { na.CC = append(na.CC, f) } - var activity *activitystreams.Activity if isUpdate { activity = activitystreams.NewUpdateActivity(na) } else { @@ -612,6 +613,28 @@ func federatePost(app *App, p *PublicPost, collID int64, isUpdate bool) error { log.Error("Couldn't post! %v", err) } } + + for _, tag := range na.Tag { + if tag.Type == "Mention" { + activity = activitystreams.NewCreateActivity(na) + activity.To = na.To + activity.CC = na.CC + // This here might be redundant in some cases as we might have already + // sent this to the sharedInbox of this instance above, but we need too + // much logic to catch this at the expense of the odd extra request. + // I don't believe we'd ever have too many mentions in a single post that this + // could become a burden. + remoteActor, err := activityserve.NewRemoteActor(tag.HRef) + if err != nil { + log.Error("Couldn't fetch remote actor", err) + } + err = makeActivityPost(app.cfg.App.Host, actor, remoteActor.GetInbox(), activity) + if err != nil { + log.Error("Couldn't post! %v", err) + } + } + } + return nil } diff --git a/go.mod b/go.mod index 9c67aeb..6711795 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,13 @@ require ( github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect github.com/captncraig/cors v0.0.0-20180620154129-376d45073b49 // indirect github.com/clbanning/mxj v1.8.4 // indirect + github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 // indirect github.com/dustin/go-humanize v1.0.0 github.com/fatih/color v1.7.0 github.com/go-sql-driver/mysql v1.4.1 github.com/go-test/deep v1.0.1 // indirect github.com/golang/lint v0.0.0-20181217174547-8f45f776aaf1 // indirect + github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 // indirect github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect github.com/gorilla/feeds v1.1.0 github.com/gorilla/mux v1.7.0 @@ -36,6 +38,7 @@ require ( github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c // indirect github.com/stretchr/testify v1.3.0 // indirect github.com/writeas/activity v0.1.2 + github.com/writeas/activityserve v0.0.0-20191008122325-5fc3b48e70c5 github.com/writeas/go-strip-markdown v2.0.1+incompatible github.com/writeas/go-webfinger v0.0.0-20190106002315-85cf805c86d2 github.com/writeas/httpsig v1.0.0 @@ -58,3 +61,5 @@ require ( gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 // indirect gopkg.in/yaml.v2 v2.2.2 // indirect ) + +go 1.13 diff --git a/go.sum b/go.sum index ec1e19d..356c06f 100644 --- a/go.sum +++ b/go.sum @@ -23,12 +23,15 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 h1:74lLNRzvsdIlkTgfDSMuaPjBr4cf6k7pwQQANm/yLKU= +github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/go-fed/httpsig v0.1.0 h1:6F2OxRVnNTN4OPN+Mc2jxs2WEay9/qiHT/jphlvAwIY= github.com/go-fed/httpsig v0.1.0/go.mod h1:T56HUNYZUQ1AGUzhAYPugZfp36sKApVnGBgKlIY+aIE= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -38,6 +41,8 @@ github.com/golang/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:tluoj9z5200j github.com/golang/lint v0.0.0-20181217174547-8f45f776aaf1 h1:6DVPu65tee05kY0/rciBQ47ue+AnuY8KTayV6VHikIo= github.com/golang/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY= +github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U= github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf h1:7+FW5aGwISbqUtkfmIpZJGRgNFg2ioYPvFaUxdqpDsg= github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg= @@ -115,6 +120,8 @@ github.com/tsenart/deadcode v0.0.0-20160724212837-210d2dc333e9 h1:vY5WqiEon0ZSTG github.com/tsenart/deadcode v0.0.0-20160724212837-210d2dc333e9/go.mod h1:q+QjxYvZ+fpjMXqs+XEriussHjSYqeXVnAdSV1tkMYk= github.com/writeas/activity v0.1.2 h1:Y12B5lIrabfqKE7e7HFCWiXrlfXljr9tlkFm2mp7DgY= github.com/writeas/activity v0.1.2/go.mod h1:mYYgiewmEM+8tlifirK/vl6tmB2EbjYaxwb+ndUw5T0= +github.com/writeas/activityserve v0.0.0-20191008122325-5fc3b48e70c5 h1:nG84xWpxBM8YU/FJchezJqg7yZH8ImSRow6NoYtbSII= +github.com/writeas/activityserve v0.0.0-20191008122325-5fc3b48e70c5/go.mod h1:Kz62mzYsCnrFTSTSFLXFj3fGYBQOntmBWTDDq57b46A= github.com/writeas/go-strip-markdown v2.0.1+incompatible h1:IIqxTM5Jr7RzhigcL6FkrCNfXkvbR+Nbu1ls48pXYcw= github.com/writeas/go-strip-markdown v2.0.1+incompatible/go.mod h1:Rsyu10ZhbEK9pXdk8V6MVnZmTzRG0alMNLMwa0J01fE= github.com/writeas/go-webfinger v0.0.0-20190106002315-85cf805c86d2 h1:DUsp4OhdfI+e6iUqcPQlwx8QYXuUDsToTz/x82D3Zuo= diff --git a/posts.go b/posts.go index 88730f7..520f716 100644 --- a/posts.go +++ b/posts.go @@ -1108,6 +1108,24 @@ func (p *PublicPost) ActivityObject(cfg *config.Config) *activitystreams.Object }) } } + // Find mentioned users + mentionedUsers := make(map[string]string) + //:= map[string]string{"@qwazix@pleroma.site": "https://pleroma.site/users/qwazix", "@tzo@cybre.space": "https://cybre.space/users/tzo"} + + stripper := bluemonday.StrictPolicy() + content := stripper.Sanitize(p.Content) + mentionRegex := regexp.MustCompile(`@[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,6}`) + mentions := mentionRegex.FindAllString(content, -1) + + for _, handle := range mentions { + actorIRI := RemoteLookup(handle) + mentionedUsers[handle] = actorIRI + } + + for handle, iri := range mentionedUsers { + o.CC = append(o.CC, iri) + o.Tag = append(o.Tag, activitystreams.Tag{"Mention", iri, handle}) + } return o } diff --git a/webfinger.go b/webfinger.go index c95d88e..c3f8530 100644 --- a/webfinger.go +++ b/webfinger.go @@ -11,11 +11,15 @@ package writefreely import ( + "encoding/json" + "io/ioutil" + "net/http" + "strings" + "github.com/writeas/go-webfinger" "github.com/writeas/impart" "github.com/writeas/web-core/log" "github.com/writeas/writefreely/config" - "net/http" ) type wfResolver struct { @@ -80,3 +84,28 @@ func (wfr wfResolver) DummyUser(username string, hostname string, r []webfinger. func (wfr wfResolver) IsNotFoundError(err error) bool { return err == wfUserNotFoundErr } + +// RemoteLookup looks up a user by handle at a remote server +// and returns the actor URL +// TODO make this work +func RemoteLookup(handle string) string { + handle = strings.TrimLeft(handle, "@") + // let's take the server part of the handle + parts := strings.Split(handle, "@") + resp, err := http.Get("https://" + parts[1] + "/.well-known/webfinger?resource=acct:" + handle) + if err != nil { + log.Error("Error performing webfinger request", err) + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Error("Error reading webfinger response", err) + } + + var result map[string]interface{} + json.Unmarshal(body, &result) + + aliases := result["aliases"].([]interface{}) + + return aliases[0].(string) +} From 3eb638b14a1ea89319e773a9b13240e40eb06cd3 Mon Sep 17 00:00:00 2001 From: Michael Demetriou Date: Wed, 9 Oct 2019 14:34:31 +0300 Subject: [PATCH 02/19] Fix @thebaer's comments in https://github.com/writeas/writefreely/commit/dccfae7a6137afbdb89c45026fdd1932d073c45e#commitcomment-35410380 --- activitypub.go | 16 ++++++++++++++++ posts.go | 5 ++--- webfinger.go | 3 ++- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/activitypub.go b/activitypub.go index 68ac7c9..fa25061 100644 --- a/activitypub.go +++ b/activitypub.go @@ -589,18 +589,25 @@ func federatePost(app *App, p *PublicPost, collID int64, isUpdate bool) error { inbox = f.Inbox } if _, ok := inboxes[inbox]; ok { + // check if we're already sending to this shared inbox inboxes[inbox] = append(inboxes[inbox], f.ActorID) } else { + // add the new shared inbox to the list inboxes[inbox] = []string{f.ActorID} } } var activity *activitystreams.Activity + // for each one of the shared inboxes for si, instFolls := range inboxes { + // add all followers from that instance + // to the CC field na.CC = []string{} for _, f := range instFolls { na.CC = append(na.CC, f) } + // create a new "Create" activity + // with our article as object if isUpdate { activity = activitystreams.NewUpdateActivity(na) } else { @@ -608,12 +615,19 @@ func federatePost(app *App, p *PublicPost, collID int64, isUpdate bool) error { activity.To = na.To activity.CC = na.CC } + // and post it to that sharedInbox err = makeActivityPost(app.cfg.App.Host, actor, si, activity) if err != nil { log.Error("Couldn't post! %v", err) } } + // re-create the object so that the CC list gets reset and has + // the mentioned users. This might seem wasteful but the code is + // cleaner than adding the mentioned users to CC here instead of + // in p.ActivityObject() + na = p.ActivityObject(app.cfg) + for _, tag := range na.Tag { if tag.Type == "Mention" { activity = activitystreams.NewCreateActivity(na) @@ -632,6 +646,8 @@ func federatePost(app *App, p *PublicPost, collID int64, isUpdate bool) error { if err != nil { log.Error("Couldn't post! %v", err) } + // log.Info("Activity", activity) + } } diff --git a/posts.go b/posts.go index 520f716..4b6fcc1 100644 --- a/posts.go +++ b/posts.go @@ -1110,11 +1110,10 @@ func (p *PublicPost) ActivityObject(cfg *config.Config) *activitystreams.Object } // Find mentioned users mentionedUsers := make(map[string]string) - //:= map[string]string{"@qwazix@pleroma.site": "https://pleroma.site/users/qwazix", "@tzo@cybre.space": "https://cybre.space/users/tzo"} stripper := bluemonday.StrictPolicy() content := stripper.Sanitize(p.Content) - mentionRegex := regexp.MustCompile(`@[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,6}`) + mentionRegex := regexp.MustCompile(`@[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]+\b`) mentions := mentionRegex.FindAllString(content, -1) for _, handle := range mentions { @@ -1124,7 +1123,7 @@ func (p *PublicPost) ActivityObject(cfg *config.Config) *activitystreams.Object for handle, iri := range mentionedUsers { o.CC = append(o.CC, iri) - o.Tag = append(o.Tag, activitystreams.Tag{"Mention", iri, handle}) + o.Tag = append(o.Tag, activitystreams.Tag{Type: "Mention", HRef: iri, Name: handle}) } return o } diff --git a/webfinger.go b/webfinger.go index c3f8530..d19478f 100644 --- a/webfinger.go +++ b/webfinger.go @@ -87,7 +87,6 @@ func (wfr wfResolver) IsNotFoundError(err error) bool { // RemoteLookup looks up a user by handle at a remote server // and returns the actor URL -// TODO make this work func RemoteLookup(handle string) string { handle = strings.TrimLeft(handle, "@") // let's take the server part of the handle @@ -95,11 +94,13 @@ func RemoteLookup(handle string) string { resp, err := http.Get("https://" + parts[1] + "/.well-known/webfinger?resource=acct:" + handle) if err != nil { log.Error("Error performing webfinger request", err) + return "" } body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Error("Error reading webfinger response", err) + return "" } var result map[string]interface{} From e5bbd45b49c7c6d0f17441b0a2f6fe83d7b6038f Mon Sep 17 00:00:00 2001 From: Michael Demetriou Date: Thu, 10 Oct 2019 10:59:14 +0300 Subject: [PATCH 03/19] Change the result that webfinger returns from the first alias to the last because mastodon doesn't like https://my.instance/@me but https://my.instance/users/me --- webfinger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webfinger.go b/webfinger.go index d19478f..715e1cf 100644 --- a/webfinger.go +++ b/webfinger.go @@ -108,5 +108,5 @@ func RemoteLookup(handle string) string { aliases := result["aliases"].([]interface{}) - return aliases[0].(string) + return aliases[len(aliases)-1].(string) } From 99bb77153e4d78e2ec04075ab2e7b28aae9ec2f2 Mon Sep 17 00:00:00 2001 From: Michael Demetriou Date: Thu, 10 Oct 2019 15:11:46 +0300 Subject: [PATCH 04/19] Handles are saved in `remoteusers` while the links take you to an intermediate page (WIP) that shows the user profile page url --- activitypub.go | 38 ++++++++++++++++++++++++++------------ collections.go | 15 +++++++++++++++ postrender.go | 3 +++ posts.go | 42 +++++++++++++++++++++++++++++++++++++----- routes.go | 1 + 5 files changed, 82 insertions(+), 17 deletions(-) diff --git a/activitypub.go b/activitypub.go index fa25061..081fcea 100644 --- a/activitypub.go +++ b/activitypub.go @@ -26,7 +26,6 @@ import ( "github.com/gorilla/mux" "github.com/writeas/activity/streams" - "github.com/writeas/activityserve" "github.com/writeas/httpsig" "github.com/writeas/impart" "github.com/writeas/nerds/store" @@ -45,6 +44,7 @@ type RemoteUser struct { ActorID string Inbox string SharedInbox string + Handle string } func (ru *RemoteUser) AsPerson() *activitystreams.Person { @@ -133,7 +133,7 @@ func handleFetchCollectionOutbox(app *App, w http.ResponseWriter, r *http.Reques posts, err := app.db.GetPosts(app.cfg, c, p, false, true, false) for _, pp := range *posts { pp.Collection = res - o := pp.ActivityObject(app.cfg) + o := pp.ActivityObject(app) a := activitystreams.NewCreateActivity(o) ocp.OrderedItems = append(ocp.OrderedItems, *a) } @@ -525,7 +525,7 @@ func deleteFederatedPost(app *App, p *PublicPost, collID int64) error { } p.Collection.hostName = app.cfg.App.Host actor := p.Collection.PersonObject(collID) - na := p.ActivityObject(app.cfg) + na := p.ActivityObject(app) // Add followers p.Collection.ID = collID @@ -571,7 +571,7 @@ func federatePost(app *App, p *PublicPost, collID int64, isUpdate bool) error { } } actor := p.Collection.PersonObject(collID) - na := p.ActivityObject(app.cfg) + na := p.ActivityObject(app) // Add followers p.Collection.ID = collID @@ -626,8 +626,7 @@ func federatePost(app *App, p *PublicPost, collID int64, isUpdate bool) error { // the mentioned users. This might seem wasteful but the code is // cleaner than adding the mentioned users to CC here instead of // in p.ActivityObject() - na = p.ActivityObject(app.cfg) - + na = p.ActivityObject(app) for _, tag := range na.Tag { if tag.Type == "Mention" { activity = activitystreams.NewCreateActivity(na) @@ -638,11 +637,11 @@ func federatePost(app *App, p *PublicPost, collID int64, isUpdate bool) error { // much logic to catch this at the expense of the odd extra request. // I don't believe we'd ever have too many mentions in a single post that this // could become a burden. - remoteActor, err := activityserve.NewRemoteActor(tag.HRef) - if err != nil { - log.Error("Couldn't fetch remote actor", err) - } - err = makeActivityPost(app.cfg.App.Host, actor, remoteActor.GetInbox(), activity) + + fmt.Println(tag.HRef) + fmt.Println("aaa") + remoteUser, err := getRemoteUser(app, tag.HRef) + err = makeActivityPost(app.cfg.App.Host, actor, remoteUser.Inbox, activity) if err != nil { log.Error("Couldn't post! %v", err) } @@ -656,7 +655,7 @@ func federatePost(app *App, p *PublicPost, collID int64, isUpdate bool) error { func getRemoteUser(app *App, actorID string) (*RemoteUser, error) { u := RemoteUser{ActorID: actorID} - err := app.db.QueryRow("SELECT id, inbox, shared_inbox FROM remoteusers WHERE actor_id = ?", actorID).Scan(&u.ID, &u.Inbox, &u.SharedInbox) + err := app.db.QueryRow("SELECT id, inbox, shared_inbox, handle FROM remoteusers WHERE actor_id = ?", actorID).Scan(&u.ID, &u.Inbox, &u.SharedInbox, &u.Handle) switch { case err == sql.ErrNoRows: return nil, impart.HTTPError{http.StatusNotFound, "No remote user with that ID."} @@ -668,6 +667,21 @@ func getRemoteUser(app *App, actorID string) (*RemoteUser, error) { return &u, nil } +// getRemoteUserFromHandle retrieves the profile page of a remote user +// from the @user@server.tld handle +func getRemoteUserFromHandle(app *App, handle string) (*RemoteUser, error) { + u := RemoteUser{Handle: handle} + err := app.db.QueryRow("SELECT id, actor_id, inbox, shared_inbox FROM remoteusers WHERE handle = ?", handle).Scan(&u.ID, &u.ActorID, &u.Inbox, &u.SharedInbox) + switch { + case err == sql.ErrNoRows: + return nil, impart.HTTPError{http.StatusNotFound, "No remote user with that handle."} + case err != nil: + log.Error("Couldn't get remote user %s: %v", handle, err) + return nil, err + } + return &u, nil +} + func getActor(app *App, actorIRI string) (*activitystreams.Person, *RemoteUser, error) { log.Info("Fetching actor %s locally", actorIRI) actor := &activitystreams.Person{} diff --git a/collections.go b/collections.go index c095ecb..9ce9d8e 100644 --- a/collections.go +++ b/collections.go @@ -820,6 +820,21 @@ func handleViewCollection(app *App, w http.ResponseWriter, r *http.Request) erro return err } +func handleViewMention(app *App, w http.ResponseWriter, r *http.Request) error { + vars := mux.Vars(r) + handle := vars["handle"] + + remoteUser, err := getRemoteUserFromHandle(app, handle) + if err != nil { + log.Error("Couldn't find this user in our database "+handle, err) + return err + } + + w.Write([]byte("go to " + remoteUser.ActorID)) + + return nil +} + func handleViewCollectionTag(app *App, w http.ResponseWriter, r *http.Request) error { vars := mux.Vars(r) tag := vars["tag"] diff --git a/postrender.go b/postrender.go index 83fb5ad..e31c462 100644 --- a/postrender.go +++ b/postrender.go @@ -34,6 +34,7 @@ var ( titleElementReg = regexp.MustCompile("") hashtagReg = regexp.MustCompile(`{{\[\[\|\|([^|]+)\|\|\]\]}}`) markeddownReg = regexp.MustCompile("

(.+)

") + mentionReg = regexp.MustCompile(`@[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]+\b`) ) func (p *Post) formatContent(cfg *config.Config, c *Collection, isOwner bool) { @@ -82,6 +83,8 @@ func applyMarkdownSpecial(data []byte, skipNoFollow bool, baseURL string, cfg *c tagPrefix = "/read/t/" } md = []byte(hashtagReg.ReplaceAll(md, []byte("#$1"))) + handlePrefix := baseURL + "mention:" + md = []byte(mentionReg.ReplaceAll(md, []byte("$0"))) } // Strip out bad HTML policy := getSanitizationPolicy() diff --git a/posts.go b/posts.go index 4b6fcc1..3eeb232 100644 --- a/posts.go +++ b/posts.go @@ -25,6 +25,7 @@ import ( "github.com/guregu/null/zero" "github.com/kylemcc/twitter-text-go/extract" "github.com/microcosm-cc/bluemonday" + "github.com/writeas/activityserve" stripmd "github.com/writeas/go-strip-markdown" "github.com/writeas/impart" "github.com/writeas/monday" @@ -35,7 +36,6 @@ import ( "github.com/writeas/web-core/i18n" "github.com/writeas/web-core/log" "github.com/writeas/web-core/tags" - "github.com/writeas/writefreely/config" "github.com/writeas/writefreely/page" "github.com/writeas/writefreely/parse" ) @@ -1033,7 +1033,7 @@ func fetchPost(app *App, w http.ResponseWriter, r *http.Request) error { } p.Collection = &CollectionObj{Collection: *coll} - po := p.ActivityObject(app.cfg) + po := p.ActivityObject(app) po.Context = []interface{}{activitystreams.Namespace} return impart.RenderActivityJSON(w, po, http.StatusOK) } @@ -1068,7 +1068,8 @@ func (p *PublicPost) CanonicalURL() string { return p.Collection.CanonicalURL() + p.Slug.String } -func (p *PublicPost) ActivityObject(cfg *config.Config) *activitystreams.Object { +func (p *PublicPost) ActivityObject(app *App) *activitystreams.Object { + cfg := app.cfg o := activitystreams.NewArticleObject() o.ID = p.Collection.FederatedAPIBase() + "api/posts/" + p.ID o.Published = p.Created @@ -1117,7 +1118,38 @@ func (p *PublicPost) ActivityObject(cfg *config.Config) *activitystreams.Object mentions := mentionRegex.FindAllString(content, -1) for _, handle := range mentions { - actorIRI := RemoteLookup(handle) + var actorIRI string + remoteuser, errRemoteUser := getRemoteUserFromHandle(app, handle) + if errRemoteUser != nil { + // can't find using handle in the table but the table may already have this user without + // handle from a previous version + actorIRI = RemoteLookup(handle) + _, errRemoteUser := getRemoteUser(app, actorIRI) + // if it exists then we need to update the handle + if errRemoteUser == nil { + // query := "UPDATE remoteusers SET handle='" + handle + "' WHERE actor_id='" + iri + "';" + // log.Info(query) + _, err := app.db.Exec("UPDATE remoteusers SET handle=? WHERE actor_id=?;", handle, actorIRI) + if err != nil { + log.Error("Can't update handle (" + handle + ") in database for user " + actorIRI) + } + } else { + // this probably means we don't have the user in the table so let's try to insert it + // here we need to ask the server for the inboxes + remoteActor, err := activityserve.NewRemoteActor(actorIRI) + if err != nil { + log.Error("Couldn't fetch remote actor", err) + } + fmt.Println(actorIRI, remoteActor.GetInbox(), remoteActor.GetSharedInbox(), handle) + _, err = app.db.Exec("INSERT INTO remoteusers (actor_id, inbox, shared_inbox, handle) VALUES( ?, ?, ?, ?)", actorIRI, remoteActor.GetInbox(), remoteActor.GetSharedInbox(), handle) + if err != nil { + log.Error("Can't insert remote user in database", err) + return nil + } + } + } else { + actorIRI = remoteuser.ActorID + } mentionedUsers[handle] = actorIRI } @@ -1379,7 +1411,7 @@ Are you sure it was ever here?`, return ErrCollectionPageNotFound } p.extractData() - ap := p.ActivityObject(app.cfg) + ap := p.ActivityObject(app) ap.Context = []interface{}{activitystreams.Namespace} return impart.RenderActivityJSON(w, ap, http.StatusOK) } else { diff --git a/routes.go b/routes.go index 0113e93..4c26f47 100644 --- a/routes.go +++ b/routes.go @@ -184,6 +184,7 @@ func InitRoutes(apper Apper, r *mux.Router) *mux.Router { func RouteCollections(handler *Handler, r *mux.Router) { r.HandleFunc("/page/{page:[0-9]+}", handler.Web(handleViewCollection, UserLevelReader)) + r.HandleFunc("/mention:{handle}", handler.Web(handleViewMention, UserLevelReader)) r.HandleFunc("/tag:{tag}", handler.Web(handleViewCollectionTag, UserLevelReader)) r.HandleFunc("/tag:{tag}/feed/", handler.Web(ViewFeed, UserLevelReader)) r.HandleFunc("/tags/{tag}", handler.Web(handleViewCollectionTag, UserLevelReader)) From db14f04b5900471031ad53907a20a4e64a7cff4b Mon Sep 17 00:00:00 2001 From: Michael Demetriou Date: Thu, 10 Oct 2019 16:04:43 +0300 Subject: [PATCH 05/19] Redirects from the intermediate page work and if there's an old mention there it updates the table to include the handle. migrations WIP --- collections.go | 9 +++++---- database.go | 36 ++++++++++++++++++++++++++++++++++++ migrations/migrations.go | 1 + migrations/v3.go | 23 +++++++++++++++++++++++ posts.go | 36 ++++-------------------------------- schema.sql | 1 + sqlite.sql | 1 + 7 files changed, 71 insertions(+), 36 deletions(-) create mode 100644 migrations/v3.go diff --git a/collections.go b/collections.go index 9ce9d8e..431afd1 100644 --- a/collections.go +++ b/collections.go @@ -824,13 +824,14 @@ func handleViewMention(app *App, w http.ResponseWriter, r *http.Request) error { vars := mux.Vars(r) handle := vars["handle"] - remoteUser, err := getRemoteUserFromHandle(app, handle) + remoteUser, err := app.db.getProfilePageFromHandle(app, handle) if err != nil { - log.Error("Couldn't find this user in our database "+handle, err) - return err + log.Error("Couldn't find this user "+handle, err) + return nil } - w.Write([]byte("go to " + remoteUser.ActorID)) + http.Redirect(w, r, remoteUser, http.StatusSeeOther) + w.Write([]byte("go to " + remoteUser)) return nil } diff --git a/database.go b/database.go index a3235b6..a95bfca 100644 --- a/database.go +++ b/database.go @@ -20,6 +20,7 @@ import ( "github.com/guregu/null" "github.com/guregu/null/zero" uuid "github.com/nu7hatch/gouuid" + "github.com/writeas/activityserve" "github.com/writeas/impart" "github.com/writeas/nerds/store" "github.com/writeas/web-core/activitypub" @@ -2449,3 +2450,38 @@ func handleFailedPostInsert(err error) error { log.Error("Couldn't insert into posts: %v", err) return err } + +func (db *datastore) getProfilePageFromHandle(app *App, handle string) (actorIRI string, err error) { + remoteuser, errRemoteUser := getRemoteUserFromHandle(app, handle) + if errRemoteUser != nil { + // can't find using handle in the table but the table may already have this user without + // handle from a previous version + actorIRI = RemoteLookup(handle) + _, errRemoteUser := getRemoteUser(app, actorIRI) + // if it exists then we need to update the handle + if errRemoteUser == nil { + // query := "UPDATE remoteusers SET handle='" + handle + "' WHERE actor_id='" + iri + "';" + // log.Info(query) + _, err := app.db.Exec("UPDATE remoteusers SET handle=? WHERE actor_id=?;", handle, actorIRI) + if err != nil { + log.Error("Can't update handle (" + handle + ") in database for user " + actorIRI) + } + } else { + // this probably means we don't have the user in the table so let's try to insert it + // here we need to ask the server for the inboxes + remoteActor, err := activityserve.NewRemoteActor(actorIRI) + if err != nil { + log.Error("Couldn't fetch remote actor", err) + } + fmt.Println(actorIRI, remoteActor.GetInbox(), remoteActor.GetSharedInbox(), handle) + _, err = app.db.Exec("INSERT INTO remoteusers (actor_id, inbox, shared_inbox, handle) VALUES( ?, ?, ?, ?)", actorIRI, remoteActor.GetInbox(), remoteActor.GetSharedInbox(), handle) + if err != nil { + log.Error("Can't insert remote user in database", err) + return "", err + } + } + } else { + actorIRI = remoteuser.ActorID + } + return actorIRI, nil +} diff --git a/migrations/migrations.go b/migrations/migrations.go index 70e4b7b..145c6df 100644 --- a/migrations/migrations.go +++ b/migrations/migrations.go @@ -57,6 +57,7 @@ func (m *migration) Migrate(db *datastore) error { var migrations = []Migration{ New("support user invites", supportUserInvites), // -> V1 (v0.8.0) New("support dynamic instance pages", supportInstancePages), // V1 -> V2 (v0.9.0) + New("support activityPub mentions", supportActivityPubMentions), // V2 -> V3 (v0.1x.0) } // CurrentVer returns the current migration version the application is on diff --git a/migrations/v3.go b/migrations/v3.go new file mode 100644 index 0000000..5c3f5aa --- /dev/null +++ b/migrations/v3.go @@ -0,0 +1,23 @@ +/* + * Copyright © 2019 A Bunch Tell LLC. + * + * This file is part of WriteFreely. + * + * WriteFreely is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, included + * in the LICENSE file in this source code package. + */ + +package migrations + +func supportActivityPubMentions(db *datastore) error { + t, err := db.Begin() + + _, err = t.Exec(`ALTER TABLE remoteusers ADD COLUMN handle ` + db.typeVarChar(255) + ` DEFAULT '' NOT NULL`) + if err != nil { + t.Rollback() + return err + } + + return nil +} diff --git a/posts.go b/posts.go index 3eeb232..df8be93 100644 --- a/posts.go +++ b/posts.go @@ -25,7 +25,6 @@ import ( "github.com/guregu/null/zero" "github.com/kylemcc/twitter-text-go/extract" "github.com/microcosm-cc/bluemonday" - "github.com/writeas/activityserve" stripmd "github.com/writeas/go-strip-markdown" "github.com/writeas/impart" "github.com/writeas/monday" @@ -1118,37 +1117,10 @@ func (p *PublicPost) ActivityObject(app *App) *activitystreams.Object { mentions := mentionRegex.FindAllString(content, -1) for _, handle := range mentions { - var actorIRI string - remoteuser, errRemoteUser := getRemoteUserFromHandle(app, handle) - if errRemoteUser != nil { - // can't find using handle in the table but the table may already have this user without - // handle from a previous version - actorIRI = RemoteLookup(handle) - _, errRemoteUser := getRemoteUser(app, actorIRI) - // if it exists then we need to update the handle - if errRemoteUser == nil { - // query := "UPDATE remoteusers SET handle='" + handle + "' WHERE actor_id='" + iri + "';" - // log.Info(query) - _, err := app.db.Exec("UPDATE remoteusers SET handle=? WHERE actor_id=?;", handle, actorIRI) - if err != nil { - log.Error("Can't update handle (" + handle + ") in database for user " + actorIRI) - } - } else { - // this probably means we don't have the user in the table so let's try to insert it - // here we need to ask the server for the inboxes - remoteActor, err := activityserve.NewRemoteActor(actorIRI) - if err != nil { - log.Error("Couldn't fetch remote actor", err) - } - fmt.Println(actorIRI, remoteActor.GetInbox(), remoteActor.GetSharedInbox(), handle) - _, err = app.db.Exec("INSERT INTO remoteusers (actor_id, inbox, shared_inbox, handle) VALUES( ?, ?, ?, ?)", actorIRI, remoteActor.GetInbox(), remoteActor.GetSharedInbox(), handle) - if err != nil { - log.Error("Can't insert remote user in database", err) - return nil - } - } - } else { - actorIRI = remoteuser.ActorID + actorIRI, err := app.db.getProfilePageFromHandle(app, handle) + if err != nil { + log.Info("Can't find this user either in the database nor in the remote instance") + return nil } mentionedUsers[handle] = actorIRI } diff --git a/schema.sql b/schema.sql index b3fae97..ca8ec71 100644 --- a/schema.sql +++ b/schema.sql @@ -181,6 +181,7 @@ CREATE TABLE IF NOT EXISTS `remoteusers` ( `actor_id` varchar(255) NOT NULL, `inbox` varchar(255) NOT NULL, `shared_inbox` varchar(255) NOT NULL, + `handle` varchar(255) DEFAULT '' NOT NULL, PRIMARY KEY (`id`), UNIQUE KEY `collection_id` (`actor_id`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; diff --git a/sqlite.sql b/sqlite.sql index 90989ed..e1a6b96 100644 --- a/sqlite.sql +++ b/sqlite.sql @@ -172,6 +172,7 @@ CREATE TABLE IF NOT EXISTS `remoteusers` ( actor_id TEXT NOT NULL, inbox TEXT NOT NULL, shared_inbox TEXT NOT NULL, + handle TEXT DEFAULT '' NOT NULL, CONSTRAINT collection_id UNIQUE (actor_id) ); From bc2016f00ff2b2fa58f978f489255457af4f46d5 Mon Sep 17 00:00:00 2001 From: Michael Demetriou Date: Thu, 10 Oct 2019 16:49:44 +0300 Subject: [PATCH 06/19] Fix missing commit statement in migrations/v3.go --- migrations/v3.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/migrations/v3.go b/migrations/v3.go index 5c3f5aa..c6f5012 100644 --- a/migrations/v3.go +++ b/migrations/v3.go @@ -19,5 +19,11 @@ func supportActivityPubMentions(db *datastore) error { return err } + err = t.Commit() + if err != nil { + t.Rollback() + return err + } + return nil } From b9d268982848b9d494984ab8cc3d5252762a04d9 Mon Sep 17 00:00:00 2001 From: Michael Demetriou Date: Fri, 11 Oct 2019 10:05:18 +0300 Subject: [PATCH 07/19] Fix comments on T627 pull request (https://github.com/writeas/writefreely/pull/195) --- activitypub.go | 5 ----- collections.go | 5 +---- routes.go | 4 +++- schema.sql | 1 - sqlite.sql | 1 - webfinger.go | 19 +++++++++++++++++-- 6 files changed, 21 insertions(+), 14 deletions(-) diff --git a/activitypub.go b/activitypub.go index 081fcea..a6ef1da 100644 --- a/activitypub.go +++ b/activitypub.go @@ -637,16 +637,11 @@ func federatePost(app *App, p *PublicPost, collID int64, isUpdate bool) error { // much logic to catch this at the expense of the odd extra request. // I don't believe we'd ever have too many mentions in a single post that this // could become a burden. - - fmt.Println(tag.HRef) - fmt.Println("aaa") remoteUser, err := getRemoteUser(app, tag.HRef) err = makeActivityPost(app.cfg.App.Host, actor, remoteUser.Inbox, activity) if err != nil { log.Error("Couldn't post! %v", err) } - // log.Info("Activity", activity) - } } diff --git a/collections.go b/collections.go index 431afd1..f0450b5 100644 --- a/collections.go +++ b/collections.go @@ -830,10 +830,7 @@ func handleViewMention(app *App, w http.ResponseWriter, r *http.Request) error { return nil } - http.Redirect(w, r, remoteUser, http.StatusSeeOther) - w.Write([]byte("go to " + remoteUser)) - - return nil + return impart.HTTPError{Status: http.StatusFound, Message: remoteUser} } func handleViewCollectionTag(app *App, w http.ResponseWriter, r *http.Request) error { diff --git a/routes.go b/routes.go index 4c26f47..6f6bd58 100644 --- a/routes.go +++ b/routes.go @@ -70,6 +70,9 @@ func InitRoutes(apper Apper, r *mux.Router) *mux.Router { write.HandleFunc(nodeinfo.NodeInfoPath, handler.LogHandlerFunc(http.HandlerFunc(ni.NodeInfoDiscover))) write.HandleFunc(niCfg.InfoURL, handler.LogHandlerFunc(http.HandlerFunc(ni.NodeInfo))) + // handle mentions + write.HandleFunc("/mention:{handle}", handler.Web(handleViewMention, UserLevelReader)) + // Set up dyamic page handlers // Handle auth auth := write.PathPrefix("/api/auth/").Subrouter() @@ -184,7 +187,6 @@ func InitRoutes(apper Apper, r *mux.Router) *mux.Router { func RouteCollections(handler *Handler, r *mux.Router) { r.HandleFunc("/page/{page:[0-9]+}", handler.Web(handleViewCollection, UserLevelReader)) - r.HandleFunc("/mention:{handle}", handler.Web(handleViewMention, UserLevelReader)) r.HandleFunc("/tag:{tag}", handler.Web(handleViewCollectionTag, UserLevelReader)) r.HandleFunc("/tag:{tag}/feed/", handler.Web(ViewFeed, UserLevelReader)) r.HandleFunc("/tags/{tag}", handler.Web(handleViewCollectionTag, UserLevelReader)) diff --git a/schema.sql b/schema.sql index ca8ec71..b3fae97 100644 --- a/schema.sql +++ b/schema.sql @@ -181,7 +181,6 @@ CREATE TABLE IF NOT EXISTS `remoteusers` ( `actor_id` varchar(255) NOT NULL, `inbox` varchar(255) NOT NULL, `shared_inbox` varchar(255) NOT NULL, - `handle` varchar(255) DEFAULT '' NOT NULL, PRIMARY KEY (`id`), UNIQUE KEY `collection_id` (`actor_id`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; diff --git a/sqlite.sql b/sqlite.sql index e1a6b96..90989ed 100644 --- a/sqlite.sql +++ b/sqlite.sql @@ -172,7 +172,6 @@ CREATE TABLE IF NOT EXISTS `remoteusers` ( actor_id TEXT NOT NULL, inbox TEXT NOT NULL, shared_inbox TEXT NOT NULL, - handle TEXT DEFAULT '' NOT NULL, CONSTRAINT collection_id UNIQUE (actor_id) ); diff --git a/webfinger.go b/webfinger.go index 715e1cf..72c3f95 100644 --- a/webfinger.go +++ b/webfinger.go @@ -106,7 +106,22 @@ func RemoteLookup(handle string) string { var result map[string]interface{} json.Unmarshal(body, &result) - aliases := result["aliases"].([]interface{}) + var href string + for _, link := range result["links"].([]interface{}) { + if link.(map[string]interface{})["rel"] == "self" { + href = link.(map[string]interface{})["href"].(string) + } + } - return aliases[len(aliases)-1].(string) + // if we didn't find it with the above then + // try using aliases + if href == "" { + aliases := result["aliases"].([]interface{}) + // take the last alias because mastodon has the + // https://instance.tld/@user first which + // doesn't work as an href + href = aliases[len(aliases)-1].(string) + } + + return href } From 972ec00c58252ff90e008285242881a3d0a6f886 Mon Sep 17 00:00:00 2001 From: Michael Demetriou Date: Fri, 11 Oct 2019 10:33:51 +0300 Subject: [PATCH 08/19] Update dependencies and add a comment --- go.mod | 2 +- go.sum | 2 ++ webfinger.go | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 6711795..2f66a2d 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c // indirect github.com/stretchr/testify v1.3.0 // indirect github.com/writeas/activity v0.1.2 - github.com/writeas/activityserve v0.0.0-20191008122325-5fc3b48e70c5 + github.com/writeas/activityserve v0.0.0-20191011072627-3a81f7784d5b github.com/writeas/go-strip-markdown v2.0.1+incompatible github.com/writeas/go-webfinger v0.0.0-20190106002315-85cf805c86d2 github.com/writeas/httpsig v1.0.0 diff --git a/go.sum b/go.sum index 356c06f..75626d2 100644 --- a/go.sum +++ b/go.sum @@ -122,6 +122,8 @@ github.com/writeas/activity v0.1.2 h1:Y12B5lIrabfqKE7e7HFCWiXrlfXljr9tlkFm2mp7Dg github.com/writeas/activity v0.1.2/go.mod h1:mYYgiewmEM+8tlifirK/vl6tmB2EbjYaxwb+ndUw5T0= github.com/writeas/activityserve v0.0.0-20191008122325-5fc3b48e70c5 h1:nG84xWpxBM8YU/FJchezJqg7yZH8ImSRow6NoYtbSII= github.com/writeas/activityserve v0.0.0-20191008122325-5fc3b48e70c5/go.mod h1:Kz62mzYsCnrFTSTSFLXFj3fGYBQOntmBWTDDq57b46A= +github.com/writeas/activityserve v0.0.0-20191011072627-3a81f7784d5b h1:rd2wX/bTqD55hxtBjAhwLcUgaQE36c70KX3NzpDAwVI= +github.com/writeas/activityserve v0.0.0-20191011072627-3a81f7784d5b/go.mod h1:Kz62mzYsCnrFTSTSFLXFj3fGYBQOntmBWTDDq57b46A= github.com/writeas/go-strip-markdown v2.0.1+incompatible h1:IIqxTM5Jr7RzhigcL6FkrCNfXkvbR+Nbu1ls48pXYcw= github.com/writeas/go-strip-markdown v2.0.1+incompatible/go.mod h1:Rsyu10ZhbEK9pXdk8V6MVnZmTzRG0alMNLMwa0J01fE= github.com/writeas/go-webfinger v0.0.0-20190106002315-85cf805c86d2 h1:DUsp4OhdfI+e6iUqcPQlwx8QYXuUDsToTz/x82D3Zuo= diff --git a/webfinger.go b/webfinger.go index 72c3f95..9e6f687 100644 --- a/webfinger.go +++ b/webfinger.go @@ -107,6 +107,8 @@ func RemoteLookup(handle string) string { json.Unmarshal(body, &result) var href string + // iterate over webfinger links and find the one with + // a self "rel" for _, link := range result["links"].([]interface{}) { if link.(map[string]interface{})["rel"] == "self" { href = link.(map[string]interface{})["href"].(string) From 1bda0434de893746182bd35233ae213d57b29bdc Mon Sep 17 00:00:00 2001 From: Michael Demetriou Date: Tue, 15 Oct 2019 09:59:24 +0300 Subject: [PATCH 09/19] Unmarshal to `webfinger.Resource` instead of interface{} (https://github.com/writeas/writefreely/pull/195#discussion_r334567408) --- webfinger.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/webfinger.go b/webfinger.go index 9e6f687..6fd1c59 100644 --- a/webfinger.go +++ b/webfinger.go @@ -103,26 +103,30 @@ func RemoteLookup(handle string) string { return "" } - var result map[string]interface{} - json.Unmarshal(body, &result) + var result webfinger.Resource + err = json.Unmarshal(body, &result) + + if err != nil { + log.Error("Unsupported webfinger response received", err) + return "" + } var href string // iterate over webfinger links and find the one with // a self "rel" - for _, link := range result["links"].([]interface{}) { - if link.(map[string]interface{})["rel"] == "self" { - href = link.(map[string]interface{})["href"].(string) + for _, link := range result.Links { + if link.Rel == "self" { + href = link.HRef } } // if we didn't find it with the above then // try using aliases if href == "" { - aliases := result["aliases"].([]interface{}) // take the last alias because mastodon has the // https://instance.tld/@user first which // doesn't work as an href - href = aliases[len(aliases)-1].(string) + href = result.Aliases[len(result.Aliases)-1] } return href From bbb7b281109dbdbac239c29b8c2d3ecb9e550f77 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Tue, 26 Nov 2019 13:32:33 -0500 Subject: [PATCH 10/19] Bump Travis build to Go 1.12 This fixes the `undefined: strings.ReplaceAll` build error. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1e58d6b..c2ebadb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: go go: - - "1.11.x" + - "1.12.x" env: - GO111MODULE=on From 181af8c5c8bd07a06359ff375e9e335e9d6db602 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 27 Nov 2019 16:37:52 -0500 Subject: [PATCH 11/19] Update httpsig and activityserve This fixes activityserve crashes caused by mentioning WriteFreely instances. --- go.mod | 6 ++---- go.sum | 6 ++++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 5ac4a8b..9c02895 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 // indirect github.com/dustin/go-humanize v1.0.0 github.com/fatih/color v1.7.0 + github.com/go-fed/httpsig v0.1.1-0.20190924171022-f4c36041199d // indirect github.com/go-sql-driver/mysql v1.4.1 github.com/go-test/deep v1.0.1 // indirect github.com/golang/lint v0.0.0-20181217174547-8f45f776aaf1 // indirect @@ -33,19 +34,17 @@ require ( github.com/pelletier/go-toml v1.2.0 // indirect github.com/pkg/errors v0.8.1 // indirect github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be // indirect - github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 // indirect github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c // indirect github.com/stretchr/testify v1.3.0 // indirect github.com/writeas/activity v0.1.2 - github.com/writeas/activityserve v0.0.0-20191011072627-3a81f7784d5b + github.com/writeas/activityserve v0.0.0-20191115095800-dd6d19cc8b89 github.com/writeas/go-strip-markdown v2.0.1+incompatible github.com/writeas/go-webfinger v0.0.0-20190106002315-85cf805c86d2 github.com/writeas/httpsig v1.0.0 github.com/writeas/impart v1.1.0 github.com/writeas/monday v0.0.0-20181024183321-54a7dd579219 github.com/writeas/nerds v1.0.0 - github.com/writeas/openssl-go v1.0.0 // indirect github.com/writeas/saturday v1.7.1 github.com/writeas/slug v1.2.0 github.com/writeas/web-core v1.2.0 @@ -58,7 +57,6 @@ require ( google.golang.org/appengine v1.4.0 // indirect gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c // indirect gopkg.in/ini.v1 v1.41.0 - gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 // indirect gopkg.in/yaml.v2 v2.2.2 // indirect ) diff --git a/go.sum b/go.sum index c8b46ff..abebe07 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,8 @@ github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/go-fed/httpsig v0.1.0 h1:6F2OxRVnNTN4OPN+Mc2jxs2WEay9/qiHT/jphlvAwIY= github.com/go-fed/httpsig v0.1.0/go.mod h1:T56HUNYZUQ1AGUzhAYPugZfp36sKApVnGBgKlIY+aIE= +github.com/go-fed/httpsig v0.1.1-0.20190924171022-f4c36041199d h1:+uoOvOnNDgsYbWtAij4xP6Rgir3eJGjocFPxBJETU/U= +github.com/go-fed/httpsig v0.1.1-0.20190924171022-f4c36041199d/go.mod h1:T56HUNYZUQ1AGUzhAYPugZfp36sKApVnGBgKlIY+aIE= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-test/deep v1.0.1 h1:UQhStjbkDClarlmv0am7OXXO4/GaPdCGiUiMTvi28sg= @@ -124,6 +126,8 @@ github.com/writeas/activityserve v0.0.0-20191008122325-5fc3b48e70c5 h1:nG84xWpxB github.com/writeas/activityserve v0.0.0-20191008122325-5fc3b48e70c5/go.mod h1:Kz62mzYsCnrFTSTSFLXFj3fGYBQOntmBWTDDq57b46A= github.com/writeas/activityserve v0.0.0-20191011072627-3a81f7784d5b h1:rd2wX/bTqD55hxtBjAhwLcUgaQE36c70KX3NzpDAwVI= github.com/writeas/activityserve v0.0.0-20191011072627-3a81f7784d5b/go.mod h1:Kz62mzYsCnrFTSTSFLXFj3fGYBQOntmBWTDDq57b46A= +github.com/writeas/activityserve v0.0.0-20191115095800-dd6d19cc8b89 h1:NJhzq9aTccL3SSSZMrcnYhkD6sObdY9otNZ1X6/ZKNE= +github.com/writeas/activityserve v0.0.0-20191115095800-dd6d19cc8b89/go.mod h1:Kz62mzYsCnrFTSTSFLXFj3fGYBQOntmBWTDDq57b46A= github.com/writeas/go-strip-markdown v2.0.1+incompatible h1:IIqxTM5Jr7RzhigcL6FkrCNfXkvbR+Nbu1ls48pXYcw= github.com/writeas/go-strip-markdown v2.0.1+incompatible/go.mod h1:Rsyu10ZhbEK9pXdk8V6MVnZmTzRG0alMNLMwa0J01fE= github.com/writeas/go-webfinger v0.0.0-20190106002315-85cf805c86d2 h1:DUsp4OhdfI+e6iUqcPQlwx8QYXuUDsToTz/x82D3Zuo= @@ -138,6 +142,7 @@ github.com/writeas/nerds v1.0.0 h1:ZzRcCN+Sr3MWID7o/x1cr1ZbLvdpej9Y1/Ho+JKlqxo= github.com/writeas/nerds v1.0.0/go.mod h1:Gn2bHy1EwRcpXeB7ZhVmuUwiweK0e+JllNf66gvNLdU= github.com/writeas/openssl-go v1.0.0 h1:YXM1tDXeYOlTyJjoMlYLQH1xOloUimSR1WMF8kjFc5o= github.com/writeas/openssl-go v1.0.0/go.mod h1:WsKeK5jYl0B5y8ggOmtVjbmb+3rEGqSD25TppjJnETA= +github.com/writeas/saturday v1.6.0/go.mod h1:ETE1EK6ogxptJpAgUbcJD0prAtX48bSloie80+tvnzQ= github.com/writeas/saturday v1.7.1 h1:lYo1EH6CYyrFObQoA9RNWHVlpZA5iYL5Opxo7PYAnZE= github.com/writeas/saturday v1.7.1/go.mod h1:ETE1EK6ogxptJpAgUbcJD0prAtX48bSloie80+tvnzQ= github.com/writeas/slug v1.2.0 h1:EMQ+cwLiOcA6EtFwUgyw3Ge18x9uflUnOnR6bp/J+/g= @@ -150,6 +155,7 @@ github.com/writefreely/go-nodeinfo v1.2.0 h1:La+YbTCvmpTwFhBSlebWDDL81N88Qf/SCAv github.com/writefreely/go-nodeinfo v1.2.0/go.mod h1:UTvE78KpcjYOlRHupZIiSEFcXHioTXuacCbHU+CAcPg= golang.org/x/crypto v0.0.0-20180527072434-ab813273cd59 h1:hk3yo72LXLapY9EXVttc3Z1rLOxT9IuAPPX3GpY2+jo= golang.org/x/crypto v0.0.0-20180527072434-ab813273cd59/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190208162236-193df9c0f06f h1:ETU2VEl7TnT5bl7IvuKEzTDpplg5wzGYsOCAPhdoEIg= golang.org/x/crypto v0.0.0-20190208162236-193df9c0f06f/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= From ae5bbd273d55cdf808ec23869f5f1400361a090f Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 27 Nov 2019 17:54:17 -0500 Subject: [PATCH 12/19] Fix mention URL on multi-user instances Previously, links would go to /user/mention:@me@this.tld instead of /mention:@me@this.tld --- postrender.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postrender.go b/postrender.go index e31c462..0cd9f55 100644 --- a/postrender.go +++ b/postrender.go @@ -83,7 +83,7 @@ func applyMarkdownSpecial(data []byte, skipNoFollow bool, baseURL string, cfg *c tagPrefix = "/read/t/" } md = []byte(hashtagReg.ReplaceAll(md, []byte("#$1"))) - handlePrefix := baseURL + "mention:" + handlePrefix := "/mention:" md = []byte(mentionReg.ReplaceAll(md, []byte("$0"))) } // Strip out bad HTML From bb63e648835f58a3e5938b266d1da0461340e9bb Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Sat, 8 Feb 2020 12:10:47 -0500 Subject: [PATCH 13/19] Clean up getProfilePageFromHandle - Export the func - Remove commented-out code - Use log, not fmt for debug messages - Remove named return parameters - Use standard var naming schemes - Fix spacing in queries and remove unnecessary chars --- collections.go | 6 +++--- database.go | 19 ++++++++++--------- posts.go | 4 ++-- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/collections.go b/collections.go index 447d657..8ee88f1 100644 --- a/collections.go +++ b/collections.go @@ -1,5 +1,5 @@ /* - * Copyright © 2018 A Bunch Tell LLC. + * Copyright © 2018-2020 A Bunch Tell LLC. * * This file is part of WriteFreely. * @@ -861,13 +861,13 @@ func handleViewMention(app *App, w http.ResponseWriter, r *http.Request) error { vars := mux.Vars(r) handle := vars["handle"] - remoteUser, err := app.db.getProfilePageFromHandle(app, handle) + remoteUser, err := app.db.GetProfilePageFromHandle(app, handle) if err != nil { log.Error("Couldn't find this user "+handle, err) return nil } - return impart.HTTPError{Status: http.StatusFound, Message: remoteUser} + return impart.HTTPError{Status: http.StatusFound, Message: remoteUser} } func handleViewCollectionTag(app *App, w http.ResponseWriter, r *http.Request) error { diff --git a/database.go b/database.go index 7d4bfc0..eca09d7 100644 --- a/database.go +++ b/database.go @@ -2557,18 +2557,17 @@ func handleFailedPostInsert(err error) error { return err } -func (db *datastore) getProfilePageFromHandle(app *App, handle string) (actorIRI string, err error) { - remoteuser, errRemoteUser := getRemoteUserFromHandle(app, handle) - if errRemoteUser != nil { +func (db *datastore) GetProfilePageFromHandle(app *App, handle string) (string, error) { + actorIRI := "" + remoteUser, err := getRemoteUserFromHandle(app, handle) + if err != nil { // can't find using handle in the table but the table may already have this user without // handle from a previous version actorIRI = RemoteLookup(handle) _, errRemoteUser := getRemoteUser(app, actorIRI) // if it exists then we need to update the handle if errRemoteUser == nil { - // query := "UPDATE remoteusers SET handle='" + handle + "' WHERE actor_id='" + iri + "';" - // log.Info(query) - _, err := app.db.Exec("UPDATE remoteusers SET handle=? WHERE actor_id=?;", handle, actorIRI) + _, err := app.db.Exec("UPDATE remoteusers SET handle = ? WHERE actor_id = ?", handle, actorIRI) if err != nil { log.Error("Can't update handle (" + handle + ") in database for user " + actorIRI) } @@ -2579,15 +2578,17 @@ func (db *datastore) getProfilePageFromHandle(app *App, handle string) (actorIRI if err != nil { log.Error("Couldn't fetch remote actor", err) } - fmt.Println(actorIRI, remoteActor.GetInbox(), remoteActor.GetSharedInbox(), handle) - _, err = app.db.Exec("INSERT INTO remoteusers (actor_id, inbox, shared_inbox, handle) VALUES( ?, ?, ?, ?)", actorIRI, remoteActor.GetInbox(), remoteActor.GetSharedInbox(), handle) + if debugging { + log.Info("%s %s %s %s", actorIRI, remoteActor.GetInbox(), remoteActor.GetSharedInbox(), handle) + } + _, err = app.db.Exec("INSERT INTO remoteusers (actor_id, inbox, shared_inbox, handle) VALUES(?, ?, ?, ?)", actorIRI, remoteActor.GetInbox(), remoteActor.GetSharedInbox(), handle) if err != nil { log.Error("Can't insert remote user in database", err) return "", err } } } else { - actorIRI = remoteuser.ActorID + actorIRI = remoteUser.ActorID } return actorIRI, nil } diff --git a/posts.go b/posts.go index 5115c92..a918531 100644 --- a/posts.go +++ b/posts.go @@ -1,5 +1,5 @@ /* - * Copyright © 2018-2019 A Bunch Tell LLC. + * Copyright © 2018-2020 A Bunch Tell LLC. * * This file is part of WriteFreely. * @@ -1176,7 +1176,7 @@ func (p *PublicPost) ActivityObject(app *App) *activitystreams.Object { mentions := mentionRegex.FindAllString(content, -1) for _, handle := range mentions { - actorIRI, err := app.db.getProfilePageFromHandle(app, handle) + actorIRI, err := app.db.GetProfilePageFromHandle(app, handle) if err != nil { log.Info("Can't find this user either in the database nor in the remote instance") return nil From 81edb739dda0db3368349a4ef809cbc1c7bdc2a0 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Sat, 8 Feb 2020 12:19:08 -0500 Subject: [PATCH 14/19] Fix mention links by making them absolute, not relative. --- postrender.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postrender.go b/postrender.go index 1b25598..2f03f85 100644 --- a/postrender.go +++ b/postrender.go @@ -87,7 +87,7 @@ func applyMarkdownSpecial(data []byte, skipNoFollow bool, baseURL string, cfg *c tagPrefix = "/read/t/" } md = []byte(hashtagReg.ReplaceAll(md, []byte("#$1"))) - handlePrefix := "/mention:" + handlePrefix := cfg.App.Host + "/mention:" md = []byte(mentionReg.ReplaceAll(md, []byte("$0"))) } // Strip out bad HTML From 867eb53b3596bd7b3f2be3c53a3faf857f4cd36d Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Sat, 8 Feb 2020 12:55:10 -0500 Subject: [PATCH 15/19] Show 404 when remote user not found This notifies the user that the remote user doesn't exist, instead of showing a blank page. Ref T627 --- collections.go | 6 +++--- errors.go | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/collections.go b/collections.go index 8ee88f1..189b4e4 100644 --- a/collections.go +++ b/collections.go @@ -862,9 +862,9 @@ func handleViewMention(app *App, w http.ResponseWriter, r *http.Request) error { handle := vars["handle"] remoteUser, err := app.db.GetProfilePageFromHandle(app, handle) - if err != nil { - log.Error("Couldn't find this user "+handle, err) - return nil + if err != nil || remoteUser == "" { + log.Error("Couldn't find user %s: %v", handle, err) + return ErrRemoteUserNotFound } return impart.HTTPError{Status: http.StatusFound, Message: remoteUser} diff --git a/errors.go b/errors.go index c0d435c..1da713a 100644 --- a/errors.go +++ b/errors.go @@ -1,5 +1,5 @@ /* - * Copyright © 2018 A Bunch Tell LLC. + * Copyright © 2018-2020 A Bunch Tell LLC. * * This file is part of WriteFreely. * @@ -45,8 +45,9 @@ var ( ErrPostUnpublished = impart.HTTPError{Status: http.StatusGone, Message: "Post unpublished by author."} ErrPostFetchError = impart.HTTPError{Status: http.StatusInternalServerError, Message: "We encountered an error getting the post. The humans have been alerted."} - ErrUserNotFound = impart.HTTPError{http.StatusNotFound, "User doesn't exist."} - ErrUserNotFoundEmail = impart.HTTPError{http.StatusNotFound, "Please enter your username instead of your email address."} + ErrUserNotFound = impart.HTTPError{http.StatusNotFound, "User doesn't exist."} + ErrRemoteUserNotFound = impart.HTTPError{http.StatusNotFound, "Remote user not found."} + ErrUserNotFoundEmail = impart.HTTPError{http.StatusNotFound, "Please enter your username instead of your email address."} ErrUserSuspended = impart.HTTPError{http.StatusForbidden, "Account is silenced."} ) From eac223158af8e5b69d2a08044c80c1cc2b3fd114 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Sat, 8 Feb 2020 12:57:49 -0500 Subject: [PATCH 16/19] Move remote user URL to /@/ from /mention: Ref T627 --- postrender.go | 6 +++--- routes.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/postrender.go b/postrender.go index 2f03f85..4afa42a 100644 --- a/postrender.go +++ b/postrender.go @@ -38,7 +38,7 @@ var ( titleElementReg = regexp.MustCompile("") hashtagReg = regexp.MustCompile(`{{\[\[\|\|([^|]+)\|\|\]\]}}`) markeddownReg = regexp.MustCompile("

(.+)

") - mentionReg = regexp.MustCompile(`@[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]+\b`) + mentionReg = regexp.MustCompile(`@([A-Za-z0-9._%+-]+)(@[A-Za-z0-9.-]+\.[A-Za-z]+)\b`) ) func (p *Post) formatContent(cfg *config.Config, c *Collection, isOwner bool) { @@ -87,8 +87,8 @@ func applyMarkdownSpecial(data []byte, skipNoFollow bool, baseURL string, cfg *c tagPrefix = "/read/t/" } md = []byte(hashtagReg.ReplaceAll(md, []byte("#$1"))) - handlePrefix := cfg.App.Host + "/mention:" - md = []byte(mentionReg.ReplaceAll(md, []byte("$0"))) + handlePrefix := cfg.App.Host + "/@/" + md = []byte(mentionReg.ReplaceAll(md, []byte("@$1$2"))) } // Strip out bad HTML policy := getSanitizationPolicy() diff --git a/routes.go b/routes.go index a3676f6..266299b 100644 --- a/routes.go +++ b/routes.go @@ -71,7 +71,7 @@ func InitRoutes(apper Apper, r *mux.Router) *mux.Router { write.HandleFunc(niCfg.InfoURL, handler.LogHandlerFunc(http.HandlerFunc(ni.NodeInfo))) // handle mentions - write.HandleFunc("/mention:{handle}", handler.Web(handleViewMention, UserLevelReader)) + write.HandleFunc("/@/{handle}", handler.Web(handleViewMention, UserLevelReader)) configureSlackOauth(handler, write, apper.App()) configureWriteAsOauth(handler, write, apper.App()) From 457051106d9db8e739f7bfb8db878aeda9fb8e80 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Sat, 8 Feb 2020 13:04:23 -0500 Subject: [PATCH 17/19] Add u-url class and span in mention link Ref T627 --- postrender.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postrender.go b/postrender.go index 4afa42a..e70c0d5 100644 --- a/postrender.go +++ b/postrender.go @@ -88,7 +88,7 @@ func applyMarkdownSpecial(data []byte, skipNoFollow bool, baseURL string, cfg *c } md = []byte(hashtagReg.ReplaceAll(md, []byte("#$1"))) handlePrefix := cfg.App.Host + "/@/" - md = []byte(mentionReg.ReplaceAll(md, []byte("@$1$2"))) + md = []byte(mentionReg.ReplaceAll(md, []byte("@$1$2"))) } // Strip out bad HTML policy := getSanitizationPolicy() From ca4b0acf6028522b75cbf3db64d143b5eb6e100f Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Sat, 8 Feb 2020 13:05:09 -0500 Subject: [PATCH 18/19] Fix error logging format in RemoteLookup --- webfinger.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/webfinger.go b/webfinger.go index 3cf0ba7..d9976f9 100644 --- a/webfinger.go +++ b/webfinger.go @@ -1,5 +1,5 @@ /* - * Copyright © 2018 A Bunch Tell LLC. + * Copyright © 2018-2020 A Bunch Tell LLC. * * This file is part of WriteFreely. * @@ -113,9 +113,8 @@ func RemoteLookup(handle string) string { var result webfinger.Resource err = json.Unmarshal(body, &result) - if err != nil { - log.Error("Unsupported webfinger response received", err) + log.Error("Unsupported webfinger response received: %v", err) return "" } From 9589612d0e3efe61ab11493539c6d5c0b2b617d6 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Sat, 8 Feb 2020 13:05:54 -0500 Subject: [PATCH 19/19] Add TODOs for improving GetProfilePageFromHandle() --- activitypub.go | 4 ++-- database.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/activitypub.go b/activitypub.go index 80c7365..a5b140d 100644 --- a/activitypub.go +++ b/activitypub.go @@ -1,5 +1,5 @@ /* - * Copyright © 2018-2019 A Bunch Tell LLC. + * Copyright © 2018-2020 A Bunch Tell LLC. * * This file is part of WriteFreely. * @@ -715,7 +715,7 @@ func getRemoteUserFromHandle(app *App, handle string) (*RemoteUser, error) { err := app.db.QueryRow("SELECT id, actor_id, inbox, shared_inbox FROM remoteusers WHERE handle = ?", handle).Scan(&u.ID, &u.ActorID, &u.Inbox, &u.SharedInbox) switch { case err == sql.ErrNoRows: - return nil, impart.HTTPError{http.StatusNotFound, "No remote user with that handle."} + return nil, ErrRemoteUserNotFound case err != nil: log.Error("Couldn't get remote user %s: %v", handle, err) return nil, err diff --git a/database.go b/database.go index eca09d7..2d233d4 100644 --- a/database.go +++ b/database.go @@ -2563,6 +2563,7 @@ func (db *datastore) GetProfilePageFromHandle(app *App, handle string) (string, if err != nil { // can't find using handle in the table but the table may already have this user without // handle from a previous version + // TODO: Make this determination. We should know whether a user exists without a handle, or doesn't exist at all actorIRI = RemoteLookup(handle) _, errRemoteUser := getRemoteUser(app, actorIRI) // if it exists then we need to update the handle