cleanup images after 30 days if we don't need them

activity.go

@@ -237,7 +237,7 @@ func savedonk(url string, name, media string) *Donk {
 		data = img.Data
 		media = "image/" + img.Format
 	}
-	res, err := stmtSaveFile.Exec(xid, name, url, media, data)
+	res, err := stmtSaveFile.Exec(xid, name, url, media, 1, data)
 	if err != nil {
 		log.Printf("error saving file %s: %s", url, err)
 		return nil

honk.go

@@ -75,6 +75,7 @@ type Donk struct {
 	Name    string
 	URL     string
 	Media   string
+	Local   bool
 	Content []byte
 }
 
@@ -671,7 +672,7 @@ func donksforhonks(honks []*Honk) {
 		ids = append(ids, fmt.Sprintf("%d", h.ID))
 		hmap[h.ID] = h
 	}
-	q := fmt.Sprintf("select honkid, donks.fileid, xid, name, url, media from donks join files on donks.fileid = files.fileid where honkid in (%s)", strings.Join(ids, ","))
+	q := fmt.Sprintf("select honkid, donks.fileid, xid, name, url, media, local from donks join files on donks.fileid = files.fileid where honkid in (%s)", strings.Join(ids, ","))
 	rows, err := db.Query(q)
 	if err != nil {
 		log.Printf("error querying donks: %s", err)
@@ -681,7 +682,7 @@ func donksforhonks(honks []*Honk) {
 	for rows.Next() {
 		var hid int64
 		var d Donk
-		err = rows.Scan(&hid, &d.FileID, &d.XID, &d.Name, &d.URL, &d.Media)
+		err = rows.Scan(&hid, &d.FileID, &d.XID, &d.Name, &d.URL, &d.Media, &d.Local)
 		if err != nil {
 			log.Printf("error scanning donk: %s", err)
 			continue
@@ -889,7 +890,7 @@ func savehonk(w http.ResponseWriter, r *http.Request) {
 			xid += ".txt"
 		}
 		url := fmt.Sprintf("https://%s/d/%s", serverName, xid)
-		res, err := stmtSaveFile.Exec(xid, name, url, media, data)
+		res, err := stmtSaveFile.Exec(xid, name, url, media, 1, data)
 		if err != nil {
 			log.Printf("unable to save image: %s", err)
 			return
@@ -1303,8 +1304,8 @@ func cleanupdb() {
 	deadthreads := make(map[int64][]string)
 	for rows.Next() {
 		var userid int64
-        var name string
-        rows.Scan(&userid, &name)
+		var name string
+		rows.Scan(&userid, &name)
 		deadthreads[userid] = append(deadthreads[userid], name)
 	}
 	rows.Close()
@@ -1314,6 +1315,8 @@ func cleanupdb() {
 			doordie(db, "delete from honks where userid = ? and convoy = ?", userid, t)
 		}
 	}
+	expdate := time.Now().UTC().Add(-30 * 24 * time.Hour).Format(dbtimeformat)
+	doordie(db, "update files set content = '', local = 0 where length(content) > 20000 and fileid in (select fileid from donks join honks on donks.honkid = honks.honkid where honks.dt < ? and whofore = 0)", expdate)
 	doordie(db, "delete from files where fileid not in (select fileid from donks)")
 }
 
@@ -1360,8 +1363,8 @@ func prepareStatements(db *sql.DB) {
 	stmtSaveDonk = preparetodie(db, "insert into donks (honkid, fileid) values (?, ?)")
 	stmtZonkIt = preparetodie(db, "delete from honks where userid = ? and xid = ?")
 	stmtZonkDonks = preparetodie(db, "delete from donks where honkid = ?")
-	stmtFindFile = preparetodie(db, "select fileid from files where url = ?")
-	stmtSaveFile = preparetodie(db, "insert into files (xid, name, url, media, content) values (?, ?, ?, ?, ?)")
+	stmtFindFile = preparetodie(db, "select fileid from files where url = ? and local = 1")
+	stmtSaveFile = preparetodie(db, "insert into files (xid, name, url, media, local, content) values (?, ?, ?, ?, ?, ?)")
 	stmtWhatAbout = preparetodie(db, "select userid, username, displayname, about, pubkey from users where username = ?")
 	stmtSaveDub = preparetodie(db, "insert into honkers (userid, name, xid, flavor) values (?, ?, ?, ?)")
 	stmtAddDoover = preparetodie(db, "insert into doovers (dt, tries, username, rcpt, msg) values (?, ?, ?, ?, ?)")

schema.sql

@@ -1,7 +1,7 @@
 
 create table honks (honkid integer primary key, userid integer, what text, honker text, xid text, rid text, dt text, url text, audience text, noise text, convoy text, whofore integer, format text, precis text, oonker text);
 create table donks (honkid integer, fileid integer);
-create table files(fileid integer primary key, xid text, name text, url text, media text, content blob);
+create table files(fileid integer primary key, xid text, name text, url text, media text, local integer, content blob);
 create table honkers (honkerid integer primary key, userid integer, name text, xid text, flavor text, combos text, pubkey text);
 create table xonkers (xonkerid integer primary key, xid text, ibox text, obox text, sbox text, pubkey text);
 create table zonkers (zonkerid integer primary key, userid integer, name text, wherefore text);

upgradedb.go

@@ -89,6 +89,10 @@ func upgradedb() {
 		}
 		doordie(db, "update config set value = 8 where key = 'dbversion'")
 	case 8:
+		doordie(db, "alter table files add column local integer")
+		doordie(db, "update files set local = 1")
+		doordie(db, "update config set value = 9 where key = 'dbversion'")
+	case 9:
 	default:
 		log.Fatalf("can't upgrade unknown version %d", dbversion)
 	}

util.go

@@ -71,7 +71,7 @@ var dbtimeformat = "2006-01-02 15:04:05"
 var alreadyopendb *sql.DB
 var dbname = "honk.db"
 var stmtConfig *sql.Stmt
-var myVersion = 8
+var myVersion = 9
 
 func initdb() {
 	schema, err := ioutil.ReadFile("schema.sql")

views/honk.html

@@ -24,11 +24,15 @@ convoy: <a href="/t?c={{ .Convoy }}">{{ .Convoy }}</a>
 <div class="noise">
 <p>{{ .HTML }}
 {{ range .Donks }}
+{{ if .Local }}
 {{ if eq .Media "text/plain" }}
 <p><a href="/d/{{ .XID }}">Attachment: {{ .Name }}</a>
 {{ else }}
 <p><a href="/d/{{ .XID }}"><img src="/d/{{ .XID }}" title="{{ .URL }}"></a>
 {{ end }}
+{{ else }}
+<p><a href="{{ .URL }}">External Attachment: {{ .Name }}</a>
+{{ end }}
 {{ end }}
 </div>
 {{ end }}