enough of the honk filtering and censoring system (hfcs) to get going.

Ted Unangst 5 years ago
parent 336f4806f2
commit f80037d03d

@ -203,7 +203,7 @@ func iszonked(userid int64, xid string) bool {
}
func needxonk(user *WhatAbout, x *Honk) bool {
if thoudostbitethythumb(user.ID, x.Audience, x.XID) {
if rejectnote(x) {
log.Printf("not saving thumb biter? %s via %s", x.XID, x.Honker)
return false
}
@ -213,7 +213,7 @@ func needxonkid(user *WhatAbout, xid string) bool {
if strings.HasPrefix(xid, user.URL+"/") {
return false
}
if thoudostbitethythumb(user.ID, nil, xid) {
if rejectorigin(user.ID, xid) {
log.Printf("don't need thumb biter? %s", xid)
return false
}

@ -515,6 +515,7 @@ var stmtThumbBiters, stmtDeleteHonk, stmtDeleteDonks, stmtDeleteOnts, stmtSaveZo
var stmtGetZonkers, stmtRecentHonkers, stmtGetXonker, stmtSaveXonker, stmtDeleteXonker *sql.Stmt
var stmtSelectOnts, stmtSaveOnt, stmtUpdateFlags, stmtClearFlags *sql.Stmt
var stmtHonksForUserFirstClass, stmtSaveMeta, stmtDeleteMeta, stmtUpdateHonk *sql.Stmt
var stmtGetFilters *sql.Stmt
func preparetodie(db *sql.DB, s string) *sql.Stmt {
stmt, err := db.Prepare(s)
@ -581,4 +582,5 @@ func prepareStatements(db *sql.DB) {
stmtUpdateFlags = preparetodie(db, "update honks set flags = flags | ? where honkid = ?")
stmtClearFlags = preparetodie(db, "update honks set flags = flags & ~ ? where honkid = ?")
stmtSelectOnts = preparetodie(db, "select distinct(ontology) from onts join honks on onts.honkid = honks.honkid where (honks.userid = ? or honks.whofore = 2)")
stmtGetFilters = preparetodie(db, "select hfcsid, json from hfcs where userid = ?")
}

@ -2,6 +2,8 @@ changelog
-- next
+++ Add Honk Filtering and Censoring System (HFCS).
+ Inline images in received posts.
+ Times for events.

@ -53,11 +53,6 @@ You can zonk anything you like (or dislike), either your own honk or
those of others that you're tired of seeing. Be advised that deletion
works poorly in a federated environment. It's more like please disregard.
The zonkzone supports muting unwanted contacts. One may mute an actor
(zonker), a domain (zomain), thread (zonvoy), or word (zord).
Posts can be collapsed, but not hidden, by specifying a zilence regex.
-- privacy
Posted honks are public. The federated environment lacks robust privacy

159
fun.go

@ -35,7 +35,7 @@ import (
func reverbolate(userid int64, honks []*Honk) {
filt := htfilter.New()
filt.Imager = replaceimg
zilences := getzilences(userid)
zilences := getfilters(userid, filtCollapse)
for _, h := range honks {
h.What += "ed"
if h.What == "tonked" {
@ -76,7 +76,7 @@ func reverbolate(userid int64, honks []*Honk) {
h.Open = ""
}
} else {
if badword := unsee(zilences, h.Precis, h.Noise, h.Donks); badword != "" {
if badword := unsee(zilences, h); badword != "" {
if h.Precis == "" {
h.Precis = badword
}
@ -179,50 +179,6 @@ func translate(honk *Honk) {
honk.Onts = oneofakind(ontologies(honk.Noise))
}
func unsee(zilences []*regexp.Regexp, precis string, noise string, donks []*Donk) string {
for _, z := range zilences {
if z.MatchString(precis) || z.MatchString(noise) {
if precis == "" {
w := z.String()
return w[6 : len(w)-3]
}
return precis
}
for _, d := range donks {
if z.MatchString(d.Desc) {
if precis == "" {
w := z.String()
return w[6 : len(w)-3]
}
return precis
}
}
}
return ""
}
func osmosis(honks []*Honk, userid int64) []*Honk {
zords := getzords(userid)
j := 0
outer:
for _, h := range honks {
for _, z := range zords {
if z.MatchString(h.Precis) || z.MatchString(h.Noise) {
continue outer
}
for _, d := range h.Donks {
if z.MatchString(d.Desc) {
continue outer
}
}
}
honks[j] = h
j++
}
honks = honks[0:j]
return honks
}
func shortxid(xid string) string {
idx := strings.LastIndexByte(xid, '/')
if idx == -1 {
@ -695,117 +651,6 @@ func makeitworksomehowwithoutregardforkeycontinuity(keyname string, r *http.Requ
return httpsig.VerifyRequest(r, payload, zaggy)
}
var thumbbiters map[int64]map[string]bool
var zoggles map[int64]map[string]bool
var zordses map[int64][]*regexp.Regexp
var zilences map[int64][]*regexp.Regexp
var thumblock sync.Mutex
func bitethethumbs() {
rows, err := stmtThumbBiters.Query()
if err != nil {
log.Printf("error getting thumbbiters: %s", err)
return
}
defer rows.Close()
thumblock.Lock()
defer thumblock.Unlock()
thumbbiters = make(map[int64]map[string]bool)
zoggles = make(map[int64]map[string]bool)
zordses = make(map[int64][]*regexp.Regexp)
zilences = make(map[int64][]*regexp.Regexp)
for rows.Next() {
var userid int64
var name, wherefore string
err = rows.Scan(&userid, &name, &wherefore)
if err != nil {
log.Printf("error scanning zonker: %s", err)
continue
}
if wherefore == "zord" || wherefore == "zilence" {
zord := "\\b(?i:" + name + ")\\b"
re, err := regexp.Compile(zord)
if err != nil {
log.Printf("error compiling zord: %s", err)
} else {
if wherefore == "zord" {
zordses[userid] = append(zordses[userid], re)
} else {
zilences[userid] = append(zilences[userid], re)
}
}
}
if wherefore == "zoggle" {
m := zoggles[userid]
if m == nil {
m = make(map[string]bool)
zoggles[userid] = m
}
m[name] = true
}
if wherefore == "zonker" || wherefore == "zomain" {
m := thumbbiters[userid]
if m == nil {
m = make(map[string]bool)
thumbbiters[userid] = m
}
m[name] = true
}
}
}
func getzords(userid int64) []*regexp.Regexp {
thumblock.Lock()
defer thumblock.Unlock()
return zordses[userid]
}
func getzilences(userid int64) []*regexp.Regexp {
thumblock.Lock()
defer thumblock.Unlock()
return zilences[userid]
}
func thoudostbitethythumb(userid int64, who []string, objid string) bool {
thumblock.Lock()
biters := thumbbiters[userid]
thumblock.Unlock()
objwhere := originate(objid)
if objwhere != "" && biters[objwhere] {
log.Printf("thumbbiter: %s", objid)
return true
}
for _, w := range who {
if biters[w] {
log.Printf("thumbbiter: %s", w)
return true
}
where := originate(w)
if where != "" {
if biters[where] {
log.Printf("thumbbiter: %s", w)
return true
}
}
}
return false
}
func stealthmode(userid int64, r *http.Request) bool {
agent := r.UserAgent()
agent = originate(agent)
addr := r.Header.Get("X-Forwarded-For")
thumblock.Lock()
biters := thumbbiters[userid]
thumblock.Unlock()
fake := (agent != "" && biters[agent]) || (addr != "" && biters[addr])
if fake {
log.Printf("faking 404 for %s from %s", agent, addr)
}
return fake
}
func keymatch(keyname string, actor string) string {
hash := strings.IndexByte(keyname, '#')
if hash == -1 {

@ -17,16 +17,176 @@ package main
import (
"log"
"net/http"
"regexp"
)
func skipMedia(xonk *Honk) bool {
thumblock.Lock()
goggles := zoggles[xonk.UserID]
thumblock.Unlock()
type Filter struct {
ID int64
Actor string
IncludeAudience bool
Text string
IsAnnounce bool
Reject bool
SkipMedia bool
Hide bool
Collapse bool
Rewrite string
re_rewrite *regexp.Regexp
Replace string
}
type filtType uint
const (
filtNone filtType = iota
filtAny
filtReject
filtSkipMedia
filtHide
filtCollapse
filtRewrite
)
var filtNames = []string{"None", "Any", "Reject", "SkipMedia", "Hide", "Collapse", "Rewrite"}
func (ft filtType) String() string {
return filtNames[ft]
}
type afiltermap map[filtType][]*Filter
var filtcache = cacheNew(func(userid int64) (afiltermap, bool) {
rows, err := stmtGetFilters.Query(userid)
if err != nil {
log.Printf("error querying filters: %s", err)
return nil, false
}
defer rows.Close()
filtmap := make(afiltermap)
for rows.Next() {
filt := new(Filter)
var j string
var filterid int64
err = rows.Scan(&filterid, &j)
if err == nil {
err = unjsonify(j, filt)
}
if err != nil {
log.Printf("error scanning filter: %s", err)
continue
}
filt.ID = filterid
if filt.Reject {
filtmap[filtReject] = append(filtmap[filtReject], filt)
}
if filt.SkipMedia {
filtmap[filtSkipMedia] = append(filtmap[filtSkipMedia], filt)
}
if filt.Hide {
filtmap[filtHide] = append(filtmap[filtHide], filt)
}
if filt.Collapse {
filtmap[filtCollapse] = append(filtmap[filtCollapse], filt)
}
if filt.Rewrite != "" {
filtmap[filtRewrite] = append(filtmap[filtRewrite], filt)
}
}
return filtmap, true
})
func getfilters(userid int64, scope filtType) []*Filter {
var filtmap afiltermap
ok := filtcache.Get(userid, &filtmap)
if ok {
return filtmap[scope]
}
return nil
}
if goggles[xonk.Honker] || goggles[xonk.Oonker] {
log.Printf("skipping media")
return true
func rejectorigin(userid int64, origin string) bool {
if o := originate(origin); o != "" {
origin = o
}
filts := getfilters(userid, filtReject)
for _, f := range filts {
if f.Actor == origin {
log.Printf("rejecting origin: %s", origin)
return true
}
}
return false
}
func rejectactor(userid int64, actor string) bool {
origin := originate(actor)
filts := getfilters(userid, filtReject)
for _, f := range filts {
if f.Actor == actor || (origin != "" && f.Actor == origin) {
log.Printf("rejecting actor: %s", actor)
return true
}
}
return false
}
func stealthmode(userid int64, r *http.Request) bool {
agent := r.UserAgent()
agent = originate(agent)
if agent != "" {
fake := rejectorigin(userid, agent)
if fake {
log.Printf("faking 404 for %s", agent)
return fake
}
}
return false
}
func matchfilter(h *Honk, filts []*Filter) bool {
origin := originate(h.XID)
for _, f := range filts {
if f.Actor == origin || f.Actor == h.Honker {
return true
}
if f.Text != "" {
for _, d := range h.Donks {
if d.Desc == f.Text {
return true
}
}
}
}
return false
}
func rejectnote(xonk *Honk) bool {
filts := getfilters(xonk.UserID, filtReject)
return matchfilter(xonk, filts)
}
func skipMedia(xonk *Honk) bool {
filts := getfilters(xonk.UserID, filtSkipMedia)
return matchfilter(xonk, filts)
}
func unsee(filts []*Filter, h *Honk) string {
return ""
}
func osmosis(honks []*Honk, userid int64) []*Honk {
filts := getfilters(userid, filtHide)
j := 0
outer:
for _, h := range honks {
if matchfilter(h, filts) {
continue outer
}
honks[j] = h
j++
}
honks = honks[0:j]
return honks
}

@ -8,6 +8,7 @@ create table zonkers (zonkerid integer primary key, userid integer, name text, w
create table doovers(dooverid integer primary key, dt text, tries integer, username text, rcpt text, msg blob);
create table onts (ontology text, honkid integer);
create table honkmeta (honkid integer, genus text, json text);
create table hfcs (hfcsid integer primary key, userid integer, json text);
create index idx_honksxid on honks(xid);
create index idx_honksconvoy on honks(convoy);
@ -21,6 +22,7 @@ create index idx_filesurl on filemeta(url);
create index idx_ontology on onts(ontology);
create index idx_onthonkid on onts(honkid);
create index idx_honkmetaid on honkmeta(honkid);
create index idx_hfcsuser on hfcs(userid);
create table config (key text, value text);

@ -221,12 +221,11 @@ func upgradedb() {
var xid, media string
var data []byte
err = rows.Scan(&xid, &media, &data)
if err != nil {
log.Fatal(err)
if err == nil {
_, err = tx.Exec("insert into filedata (xid, media, content) values (?, ?, ?)", xid, media, data)
}
_, err = tx.Exec("insert into filedata (xid, media, content) values (?, ?, ?)", xid, media, data)
if err != nil {
log.Fatal(err)
log.Fatalf("can't save filedata: %s", err)
}
}
rows.Close()
@ -263,8 +262,10 @@ func upgradedb() {
}
for honkid, p := range places {
j, err := jsonify(p)
_, err = tx.Exec("insert into honkmeta (honkid, genus, json) values (?, ?, ?)",
honkid, "place", j)
if err == nil {
_, err = tx.Exec("insert into honkmeta (honkid, genus, json) values (?, ?, ?)",
honkid, "place", j)
}
if err != nil {
log.Fatal(err)
}
@ -276,6 +277,66 @@ func upgradedb() {
doordie(db, "update config set value = 23 where key = 'dbversion'")
fallthrough
case 23:
doordie(db, "create table hfcs (hfcsid integer primary key, userid integer, json text)")
doordie(db, "create index idx_hfcsuser on hfcs(userid)")
rows, err := db.Query("select userid, name, wherefore from zonkers where wherefore in ('zord', 'zilence', 'zoggle', 'zonker', 'zomain')")
if err != nil {
log.Fatalf("can't query zonkers: %s", err)
}
filtmap := make(map[int64][]*Filter)
for rows.Next() {
var userid int64
var name, wherefore string
err = rows.Scan(&userid, &name, &wherefore)
if err != nil {
log.Fatal("error scanning zonker: %s", err)
}
f := new(Filter)
switch wherefore {
case "zord":
f.Text = name
f.Hide = true
case "zilence":
f.Text = name
f.Collapse = true
case "zoggle":
f.Actor = name
f.SkipMedia = true
case "zonker":
f.Actor = name
f.IncludeAudience = true
f.Reject = true
case "zomain":
f.Actor = name
f.IncludeAudience = true
f.Reject = true
}
filtmap[userid] = append(filtmap[userid], f)
}
rows.Close()
tx, err := db.Begin()
if err != nil {
log.Fatalf("can't begin: %s", err)
}
for userid, filts := range filtmap {
for _, f := range filts {
j, err := jsonify(f)
if err == nil {
_, err = tx.Exec("insert into hfcs (userid, json) values (?, ?)", userid, j)
}
if err != nil {
log.Fatalf("can't save filter: %s", err)
}
}
}
err = tx.Commit()
if err != nil {
log.Fatalf("can't commit: %s", err)
}
doordie(db, "delete from zonkers where wherefore in ('zord', 'zilence', 'zoggle', 'zonker', 'zomain')")
doordie(db, "update config set value = 24 where key = 'dbversion'")
fallthrough
case 24:
default:
log.Fatalf("can't upgrade unknown version %d", dbversion)

@ -73,7 +73,7 @@ var alreadyopendb *sql.DB
var dbname = "honk.db"
var blobdbname = "blob.db"
var stmtConfig *sql.Stmt
var myVersion = 23
var myVersion = 24
func initdb() {
schema, err := ioutil.ReadFile("schema.sql")

@ -2,37 +2,15 @@
<main>
<div class="info">
<p>
<form action="/zonkzonk" method="POST">
<span class="title">it's zonking time!</span>
<input type="hidden" name="CSRF" value="{{ .ZonkCSRF }}">
<p>
<input tabindex=1 type="text" name="name" value="" autocomplete=off> - name
<p>
<input type="radio" id="iszonker" name="wherefore" value="zonker">
<label for="iszonker">Zonker</label>
<p>
<input type="radio" id="iszomain" name="wherefore" value="zomain">
<label for="iszomain">Zomain</label>
<p>
<input type="radio" id="iszonvoy" name="wherefore" value="zonvoy">
<label for="iszonvoy">Zonvoy</label>
<p>
<input type="radio" id="iszord" name="wherefore" value="zord">
<label for="iszord">Zord</label>
<p>
<input type="radio" id="iszilence" name="wherefore" value="zilence">
<label for="iszilence">Zilence</label>
<p>
<input type="radio" id="iszoggle" name="wherefore" value="zoggle">
<label for="iszoggle">Zoggle</label>
<p><br><button tabindex=1 name="zonk" value="zonk!">zonk!</button>
</form>
Work in progress
</div>
{{ $zonkcsrf := .ZonkCSRF }}
{{ range .Zonkers }}
{{ range $how, $filters := .Filters }}
{{ range $filters }}
<section class="honk">
<p>What: {{ .Name }}
<p>Where: {{ .Wherefore }}
<p>How: {{ $how }}
{{ with .Actor }}<p>Who: {{ . }}{{ end }}
{{ with .Text }}<p>What: {{ . }}{{ end }}
<form action="/zonkzonk" method="POST">
<input type="hidden" name="CSRF" value="{{ $zonkcsrf }}">
<input type="hidden" name="zonkerid" value="{{ .ID }}">
@ -42,4 +20,5 @@
<p>
</section>
{{ end }}
{{ end }}
</main>

@ -307,8 +307,7 @@ func inbox(w http.ResponseWriter, r *http.Request) {
log.Printf("keyname actor mismatch: %s <> %s", keyname, who)
return
}
objid, _ := j.GetString("id")
if thoudostbitethythumb(user.ID, []string{who}, objid) {
if rejectactor(user.ID, who) {
log.Printf("ignoring thumb sucker %s", who)
return
}
@ -1240,37 +1239,14 @@ func submithonker(w http.ResponseWriter, r *http.Request) {
func zonkzone(w http.ResponseWriter, r *http.Request) {
userinfo := login.GetUserInfo(r)
rows, err := stmtGetZonkers.Query(userinfo.UserID)
if err != nil {
log.Printf("err: %s", err)
return
}
defer rows.Close()
var zonkers []Zonker
for rows.Next() {
var z Zonker
rows.Scan(&z.ID, &z.Name, &z.Wherefore)
zonkers = append(zonkers, z)
}
sort.Slice(zonkers, func(i, j int) bool {
w1 := zonkers[i].Wherefore
w2 := zonkers[j].Wherefore
if w1 == w2 {
return zonkers[i].Name < zonkers[j].Name
}
if w1 == "zonvoy" {
w1 = "zzzzzzz"
}
if w2 == "zonvoy" {
w2 = "zzzzzzz"
}
return w1 < w2
})
var filters afiltermap
filtcache.Get(userinfo.UserID, &filters)
templinfo := getInfo(r)
templinfo["Zonkers"] = zonkers
templinfo["Filters"] = filters
templinfo["ZonkCSRF"] = login.GetCSRF("zonkzonk", r)
err = readviews.Execute(w, "zonkers.html", templinfo)
err := readviews.Execute(w, "zonkers.html", templinfo)
if err != nil {
log.Print(err)
}
@ -1282,9 +1258,9 @@ func zonkzonk(w http.ResponseWriter, r *http.Request) {
if itsok == "iforgiveyou" {
zonkerid, _ := strconv.ParseInt(r.FormValue("zonkerid"), 10, 0)
db := opendatabase()
db.Exec("delete from zonkers where userid = ? and zonkerid = ?",
db.Exec("delete from hfcs where userid = ? and hfcsid = ?",
userinfo.UserID, zonkerid)
bitethethumbs()
filtcache.Clear(userinfo.UserID)
http.Redirect(w, r, "/zonkzone", http.StatusSeeOther)
return
}
@ -1308,7 +1284,7 @@ func zonkzonk(w http.ResponseWriter, r *http.Request) {
userinfo.UserID, name, wherefore)
if wherefore != "zonvoy" {
bitethethumbs()
filtcache.Clear(userinfo.UserID)
}
http.Redirect(w, r, "/zonkzone", http.StatusSeeOther)
@ -1550,8 +1526,6 @@ func serve() {
}
}
bitethethumbs()
mux := mux.NewRouter()
mux.Use(login.Checker)

Loading…
Cancel
Save