From 89aa0a821da708ce69cee608905e4e7e99a28fac Mon Sep 17 00:00:00 2001 From: Ted Unangst Date: Mon, 11 Nov 2019 18:30:40 -0500 Subject: [PATCH 1/4] put a limit on some caches that may grow kinda large --- activity.go | 2 +- fun.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/activity.go b/activity.go index 7cf62ac..71b80d6 100644 --- a/activity.go +++ b/activity.go @@ -1158,7 +1158,7 @@ var oldjonks = cache.New(cache.Options{Filler: func(xid string) ([]byte, bool) { j["@context"] = itiswhatitis return j.ToBytes(), true -}}) +}, Limit: 128}) func gimmejonk(xid string) ([]byte, bool) { var j []byte diff --git a/fun.go b/fun.go index 142dac0..b3ecf33 100644 --- a/fun.go +++ b/fun.go @@ -603,7 +603,7 @@ var zaggies = cache.New(cache.Options{Filler: func(keyname string) (*rsa.PublicK return key, true } return nil, true -}}) +}, Limit: 512}) func zaggy(keyname string) *rsa.PublicKey { var key *rsa.PublicKey From 40540438d347cc47acca803fd7354f84a4b54abc Mon Sep 17 00:00:00 2001 From: Ted Unangst Date: Tue, 12 Nov 2019 01:09:08 -0500 Subject: [PATCH 2/4] add an api command to gethonks --- database.go | 1 + docs/honk.3 | 19 +++++++++++ toys/Makefile | 5 ++- toys/README | 6 ++++ toys/youvegothonks.go | 79 +++++++++++++++++++++++++++++++++++++++++++ web.go | 46 +++++++++++++++++++++++++ 6 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 toys/youvegothonks.go diff --git a/database.go b/database.go index dbe1b76..fa8afc2 100644 --- a/database.go +++ b/database.go @@ -474,6 +474,7 @@ func savehonk(h *Honk) error { if err != nil { log.Printf("error saving honk: %s", err) } + honkhonkline() return err } diff --git a/docs/honk.3 b/docs/honk.3 index 5f0ffd0..f9124c2 100644 --- a/docs/honk.3 +++ b/docs/honk.3 @@ -87,6 +87,25 @@ The ActivityPub ID that this honk is in reply to. .El .Pp Upon success, the honk action will return the URL for the created honk. +.Ss gethonks +The +.Dq gethonks +.Fa action +can be used to query for honks. +The following parameters are used. +.Bl -tag -width placename +.It Fa page +Should be one of +.Dq home +or +.Dq atme . +.It Fa after +Only return honks after the specified ID. +.It Fa wait +If there are no results, wait this many seconds for something to appear. +.El +.Pp +The result will be returned as json. .Sh EXAMPLES Refer to the sample code in the .Pa toys diff --git a/toys/Makefile b/toys/Makefile index 3c288ad..c2c2f3b 100644 --- a/toys/Makefile +++ b/toys/Makefile @@ -1,8 +1,11 @@ -all: gettoken saytheday +all: gettoken saytheday youvegothonks gettoken: gettoken.go go build gettoken.go saytheday: saytheday.go go build saytheday.go + +youvegothonks: youvegothonks.go + go build youvegothonks.go diff --git a/toys/README b/toys/README index ea03164..9239b9a 100644 --- a/toys/README +++ b/toys/README @@ -1,3 +1,9 @@ These are all standalone programs, meant to be compiled individually. A little of this, a little of that. + +gettoken.go - obtains an authorization token + +saytheday.go - posts a new honk + +youvegothonks.go - polls for new mesages diff --git a/toys/youvegothonks.go b/toys/youvegothonks.go new file mode 100644 index 0000000..94cb9eb --- /dev/null +++ b/toys/youvegothonks.go @@ -0,0 +1,79 @@ +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "log" + "encoding/json" + "net/http" + "net/url" + "os" + "time" +) + +type Honk struct { + ID int + Honker string + Noise string +} + +type HonkSet struct { + Honks []Honk +} + +func gethonks(server, token string, wanted int) HonkSet { + form := make(url.Values) + form.Add("action", "gethonks") + form.Add("page", "atme") + form.Add("after", fmt.Sprintf("%d", wanted)) + form.Add("wait", "30") + apiurl := fmt.Sprintf("https://%s/api?%s", server, form.Encode()) + req, err := http.NewRequest("GET", apiurl, nil) + if err != nil { + log.Fatal(err) + } + req.Header.Add("Authorization", token) + resp, err := http.DefaultClient.Do(req) + if err != nil { + log.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + answer, _ := ioutil.ReadAll(resp.Body) + log.Fatalf("status: %d: %s", resp.StatusCode, answer) + } + var honks HonkSet + d := json.NewDecoder(resp.Body) + err = d.Decode(&honks) + if err != nil { + log.Fatal(err) + } + return honks +} + +func main() { + server := "" + token := "" + flag.StringVar(&server, "server", server, "server to connnect") + flag.StringVar(&token, "token", token, "auth token to use") + flag.Parse() + + if server == "" || token == "" { + flag.Usage() + os.Exit(1) + } + + wanted := 0 + + for { + honks := gethonks(server, token, wanted) + for _, h := range honks.Honks { + fmt.Printf("you've got a honk from %s\n%s\n", h.Honker, h.Noise) + if wanted < h.ID { + wanted = h.ID + } + } + time.Sleep(1 * time.Second) + } +} diff --git a/web.go b/web.go index 353331e..9ddbcb3 100644 --- a/web.go +++ b/web.go @@ -2038,15 +2038,61 @@ func webhydra(w http.ResponseWriter, r *http.Request) { } } +var honkline = make(chan bool) + +func honkhonkline() { + for { + select { + case honkline <- true: + default: + return + } + } +} + func apihandler(w http.ResponseWriter, r *http.Request) { u := login.GetUserInfo(r) + userid := u.UserID action := r.FormValue("action") + wait, _ := strconv.ParseInt(r.FormValue("wait"), 10, 0) log.Printf("api request '%s' on behalf of %s", action, u.Username) switch action { case "honk": submithonk(w, r, true) + case "gethonks": + var honks []*Honk + wanted, _ := strconv.ParseInt(r.FormValue("after"), 10, 0) + page := r.FormValue("page") + var waitchan <-chan time.Time + requery: + switch page { + case "atme": + honks = gethonksforme(userid, wanted) + honks = osmosis(honks, userid, false) + case "home": + honks = gethonksforuser(userid, wanted) + honks = osmosis(honks, userid, true) + default: + http.Error(w, "unknown page", http.StatusNotFound) + return + } + if len(honks) == 0 && wait > 0 { + if waitchan == nil { + waitchan = time.After(time.Duration(wait) * time.Second) + } + select { + case <-honkline: + goto requery + case <-waitchan: + } + } + reverbolate(userid, honks) + j := junk.New() + j["honks"] = honks + j.Write(w) default: http.Error(w, "unknown action", http.StatusNotFound) + return } } From 0c0d44f4449002135913f4011ce94330ceaf1558 Mon Sep 17 00:00:00 2001 From: Ted Unangst Date: Tue, 12 Nov 2019 01:10:38 -0500 Subject: [PATCH 3/4] mention Authorization header --- docs/honk.3 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/honk.3 b/docs/honk.3 index f9124c2..c12600e 100644 --- a/docs/honk.3 +++ b/docs/honk.3 @@ -33,6 +33,9 @@ The desired action. See below. .It Fa token An authorization token. +Alternatively, may be passed in the +.Dq Authorization +HTTP header. .El .Pp The API URL for all actions other than login and logout is From 0a03f937ecce58ba47b364bc1a0d8ffc93e7593b Mon Sep 17 00:00:00 2001 From: Ted Unangst Date: Tue, 12 Nov 2019 01:43:16 -0500 Subject: [PATCH 4/4] reduce number of honks on some pages --- database.go | 15 +++++++++------ web.go | 3 --- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/database.go b/database.go index fa8afc2..3706cde 100644 --- a/database.go +++ b/database.go @@ -166,11 +166,11 @@ func getbonk(userid int64, xid string) *Honk { func getpublichonks() []*Honk { dt := time.Now().UTC().Add(-7 * 24 * time.Hour).Format(dbtimeformat) - rows, err := stmtPublicHonks.Query(dt) + rows, err := stmtPublicHonks.Query(dt, 25) return getsomehonks(rows, err) } func geteventhonks(userid int64) []*Honk { - rows, err := stmtEventHonks.Query(userid) + rows, err := stmtEventHonks.Query(userid, 25) honks := getsomehonks(rows, err) sort.Slice(honks, func(i, j int) bool { var t1, t2 time.Time @@ -199,10 +199,12 @@ func geteventhonks(userid int64) []*Honk { func gethonksbyuser(name string, includeprivate bool, wanted int64) []*Honk { dt := time.Now().UTC().Add(-7 * 24 * time.Hour).Format(dbtimeformat) whofore := 2 + limit := 25 if includeprivate { whofore = 3 + limit = 50 } - rows, err := stmtUserHonks.Query(wanted, whofore, name, dt) + rows, err := stmtUserHonks.Query(wanted, whofore, name, dt, limit) return getsomehonks(rows, err) } func gethonksforuser(userid int64, wanted int64) []*Honk { @@ -712,13 +714,14 @@ func prepareStatements(db *sql.DB) { selecthonks := "select honks.honkid, honks.userid, username, what, honker, oonker, honks.xid, rid, dt, url, audience, noise, precis, format, convoy, whofore, flags from honks join users on honks.userid = users.userid " limit := " order by honks.honkid desc limit 250" + smalllimit := " order by honks.honkid desc limit ?" butnotthose := " and convoy not in (select name from zonkers where userid = ? and wherefore = 'zonvoy' order by zonkerid desc limit 100)" stmtOneXonk = preparetodie(db, selecthonks+"where honks.userid = ? and xid = ?") stmtAnyXonk = preparetodie(db, selecthonks+"where xid = ? order by honks.honkid asc") stmtOneBonk = preparetodie(db, selecthonks+"where honks.userid = ? and xid = ? and what = 'bonk' and whofore = 2") - stmtPublicHonks = preparetodie(db, selecthonks+"where whofore = 2 and dt > ?"+limit) - stmtEventHonks = preparetodie(db, selecthonks+"where (whofore = 2 or honks.userid = ?) and what = 'event'"+limit) - stmtUserHonks = preparetodie(db, selecthonks+"where honks.honkid > ? and (whofore = 2 or whofore = ?) and username = ? and dt > ?"+limit) + stmtPublicHonks = preparetodie(db, selecthonks+"where whofore = 2 and dt > ?"+smalllimit) + stmtEventHonks = preparetodie(db, selecthonks+"where (whofore = 2 or honks.userid = ?) and what = 'event'"+smalllimit) + stmtUserHonks = preparetodie(db, selecthonks+"where honks.honkid > ? and (whofore = 2 or whofore = ?) and username = ? and dt > ?"+smalllimit) myhonkers := " and honker in (select xid from honkers where userid = ? and (flavor = 'sub' or flavor = 'peep' or flavor = 'presub') and combos not like '% - %')" stmtHonksForUser = preparetodie(db, selecthonks+"where honks.honkid > ? and honks.userid = ? and dt > ?"+myhonkers+butnotthose+limit) stmtHonksForUserFirstClass = preparetodie(db, selecthonks+"where honks.honkid > ? and honks.userid = ? and dt > ? and (what <> 'tonk')"+myhonkers+butnotthose+limit) diff --git a/web.go b/web.go index 9ddbcb3..74c6fb0 100644 --- a/web.go +++ b/web.go @@ -161,9 +161,6 @@ func showrss(w http.ResponseWriter, r *http.Request) { } else { honks = getpublichonks() } - if len(honks) > 20 { - honks = honks[0:20] - } reverbolate(-1, honks) home := fmt.Sprintf("https://%s/", serverName)