From cabc3e94b440a15f5962520583d77beb77026209 Mon Sep 17 00:00:00 2001 From: ston1th Date: Sun, 24 Nov 2019 12:25:19 +0100 Subject: [PATCH] added stats page and password reset --- pkg/cmd/cmd.go | 43 +- pkg/cmd/helper.go | 14 + pkg/core/const.go | 1 + pkg/core/types.go | 8 + pkg/db/db.go | 16 +- pkg/fs/helper.go | 1 + pkg/index/index.go | 5 + pkg/server/handler.go | 51 +- pkg/server/helper.go | 68 + pkg/server/routes.go | 7 + pkg/server/templates.go | 38 +- templates.sh | 2 + templates/menu.html | 1 + templates/stats.html | 33 + templates/userEdit.html | 2 +- .../git.giftfish.de/ston1th/authdav/LICENSE | 24 + .../git.giftfish.de/ston1th/authdav/README.md | 59 + .../ston1th/authdav/authdav.go | 57 + .../git.giftfish.de/ston1th/authdav/file.go | 46 + .../ston1th/authdav/fileinfo.go | 44 + .../ston1th/authdav/filesystem.go | 79 + .../git.giftfish.de/ston1th/authdav/filter.go | 50 + vendor/git.giftfish.de/ston1th/authdav/go.mod | 5 + vendor/git.giftfish.de/ston1th/authdav/go.sum | 5 + .../blevesearch/segment/maketesttables.go | 219 -- vendor/golang.org/x/net/AUTHORS | 3 + vendor/golang.org/x/net/CONTRIBUTORS | 3 + vendor/golang.org/x/net/LICENSE | 27 + vendor/golang.org/x/net/PATENTS | 22 + vendor/golang.org/x/net/webdav/file.go | 795 +++++++ vendor/golang.org/x/net/webdav/if.go | 173 ++ .../x/net/webdav/internal/xml/README | 11 + .../x/net/webdav/internal/xml/marshal.go | 1223 ++++++++++ .../x/net/webdav/internal/xml/read.go | 692 ++++++ .../x/net/webdav/internal/xml/typeinfo.go | 371 +++ .../x/net/webdav/internal/xml/xml.go | 1998 +++++++++++++++++ vendor/golang.org/x/net/webdav/lock.go | 445 ++++ vendor/golang.org/x/net/webdav/prop.go | 469 ++++ vendor/golang.org/x/net/webdav/webdav.go | 706 ++++++ vendor/golang.org/x/net/webdav/xml.go | 519 +++++ vendor/golang.org/x/sys/unix/mkasm_darwin.go | 61 - vendor/golang.org/x/sys/unix/mkpost.go | 106 - vendor/golang.org/x/sys/unix/mksyscall.go | 407 ---- .../x/sys/unix/mksyscall_aix_ppc.go | 404 ---- .../x/sys/unix/mksyscall_aix_ppc64.go | 602 ----- .../x/sys/unix/mksyscall_solaris.go | 335 --- vendor/golang.org/x/sys/unix/mksysnum.go | 190 -- vendor/golang.org/x/sys/unix/types_aix.go | 236 -- vendor/golang.org/x/sys/unix/types_darwin.go | 283 --- .../golang.org/x/sys/unix/types_dragonfly.go | 263 --- vendor/golang.org/x/sys/unix/types_freebsd.go | 356 --- vendor/golang.org/x/sys/unix/types_netbsd.go | 289 --- vendor/golang.org/x/sys/unix/types_openbsd.go | 282 --- vendor/golang.org/x/sys/unix/types_solaris.go | 266 --- vendor/modules.txt | 41 +- 55 files changed, 8094 insertions(+), 4362 deletions(-) create mode 100644 pkg/cmd/helper.go create mode 100644 pkg/server/helper.go create mode 100644 templates/stats.html create mode 100644 vendor/git.giftfish.de/ston1th/authdav/LICENSE create mode 100644 vendor/git.giftfish.de/ston1th/authdav/README.md create mode 100644 vendor/git.giftfish.de/ston1th/authdav/authdav.go create mode 100644 vendor/git.giftfish.de/ston1th/authdav/file.go create mode 100644 vendor/git.giftfish.de/ston1th/authdav/fileinfo.go create mode 100644 vendor/git.giftfish.de/ston1th/authdav/filesystem.go create mode 100644 vendor/git.giftfish.de/ston1th/authdav/filter.go create mode 100644 vendor/git.giftfish.de/ston1th/authdav/go.mod create mode 100644 vendor/git.giftfish.de/ston1th/authdav/go.sum delete mode 100644 vendor/github.com/blevesearch/segment/maketesttables.go create mode 100644 vendor/golang.org/x/net/AUTHORS create mode 100644 vendor/golang.org/x/net/CONTRIBUTORS create mode 100644 vendor/golang.org/x/net/LICENSE create mode 100644 vendor/golang.org/x/net/PATENTS create mode 100644 vendor/golang.org/x/net/webdav/file.go create mode 100644 vendor/golang.org/x/net/webdav/if.go create mode 100644 vendor/golang.org/x/net/webdav/internal/xml/README create mode 100644 vendor/golang.org/x/net/webdav/internal/xml/marshal.go create mode 100644 vendor/golang.org/x/net/webdav/internal/xml/read.go create mode 100644 vendor/golang.org/x/net/webdav/internal/xml/typeinfo.go create mode 100644 vendor/golang.org/x/net/webdav/internal/xml/xml.go create mode 100644 vendor/golang.org/x/net/webdav/lock.go create mode 100644 vendor/golang.org/x/net/webdav/prop.go create mode 100644 vendor/golang.org/x/net/webdav/webdav.go create mode 100644 vendor/golang.org/x/net/webdav/xml.go delete mode 100644 vendor/golang.org/x/sys/unix/mkasm_darwin.go delete mode 100644 vendor/golang.org/x/sys/unix/mkpost.go delete mode 100644 vendor/golang.org/x/sys/unix/mksyscall.go delete mode 100644 vendor/golang.org/x/sys/unix/mksyscall_aix_ppc.go delete mode 100644 vendor/golang.org/x/sys/unix/mksyscall_aix_ppc64.go delete mode 100644 vendor/golang.org/x/sys/unix/mksyscall_solaris.go delete mode 100644 vendor/golang.org/x/sys/unix/mksysnum.go delete mode 100644 vendor/golang.org/x/sys/unix/types_aix.go delete mode 100644 vendor/golang.org/x/sys/unix/types_darwin.go delete mode 100644 vendor/golang.org/x/sys/unix/types_dragonfly.go delete mode 100644 vendor/golang.org/x/sys/unix/types_freebsd.go delete mode 100644 vendor/golang.org/x/sys/unix/types_netbsd.go delete mode 100644 vendor/golang.org/x/sys/unix/types_openbsd.go delete mode 100644 vendor/golang.org/x/sys/unix/types_solaris.go diff --git a/pkg/cmd/cmd.go b/pkg/cmd/cmd.go index 33339d9..c1a41de 100644 --- a/pkg/cmd/cmd.go +++ b/pkg/cmd/cmd.go @@ -5,6 +5,7 @@ package cmd import ( "encoding/json" "errors" + "fmt" "git.giftfish.de/ston1th/docstore/pkg/core" "git.giftfish.de/ston1th/docstore/pkg/db" "git.giftfish.de/ston1th/docstore/pkg/log" @@ -174,9 +175,43 @@ func Run(version string) { return nil }, }, + { + Name: "reset", + Usage: "reset the users password and TOTP token (docstore must be stopped)", + Flags: defaultFlags(nil), + Action: func(c *cli.Context) error { + if err := loadConfig(); err != nil { + stdlog.Fatal("reset: ", err) + } + DB, err := db.NewPlain(config) + if err != nil { + stdlog.Fatal("reset: ", err) + } + u, err := DB.GetUser() + if err != nil { + stdlog.Fatal("reset: ", err) + } + fmt.Println("username:", u.Username) + pw, err := password() + if err != nil { + stdlog.Fatal("reset: ", err) + } + if err = DB.UpdateUserPassword(pw); err != nil { + stdlog.Fatal("reset: ", err) + } + fmt.Println("password:", pw) + if err = DB.UpdateUserSecret(""); err != nil { + stdlog.Fatal("reset: ", err) + } + if err = DB.Close(); err != nil { + stdlog.Fatal("reset: ", err) + } + return nil + }, + }, { Name: "dump", - Usage: "create a json dump of all users (docstore needs to be stopped)", + Usage: "create a json dump of all users (docstore must be stopped)", Flags: defaultFlags(dumpRestoreFlags()), Action: func(c *cli.Context) error { var ( @@ -192,7 +227,7 @@ func Run(version string) { stdlog.Fatal("dump: ", err) } } - DB, err := db.New(config) + DB, err := db.NewPlain(config) if err != nil { stdlog.Fatal("dump: ", err) } @@ -209,7 +244,7 @@ func Run(version string) { }, { Name: "restore", - Usage: "restore a json dump (docstore needs to be stopped)", + Usage: "restore a json dump (docstore must be stopped)", Flags: defaultFlags(dumpRestoreFlags()), Action: func(c *cli.Context) error { var ( @@ -225,7 +260,7 @@ func Run(version string) { stdlog.Fatal("restore: ", err) } } - DB, err := db.New(config) + DB, err := db.NewPlain(config) if err != nil { stdlog.Fatal("restore: ", err) } diff --git a/pkg/cmd/helper.go b/pkg/cmd/helper.go new file mode 100644 index 0000000..f13593d --- /dev/null +++ b/pkg/cmd/helper.go @@ -0,0 +1,14 @@ +// Copyright (C) 2019 Marius Schellenberger + +package cmd + +import ( + "crypto/rand" + "encoding/hex" +) + +func password() (string, error) { + b := make([]byte, 16) + _, err := rand.Read(b) + return hex.EncodeToString(b), err +} diff --git a/pkg/core/const.go b/pkg/core/const.go index 9cf77cc..4268a1b 100644 --- a/pkg/core/const.go +++ b/pkg/core/const.go @@ -10,6 +10,7 @@ const ( SearchURI = "/search" LogsURI = "/logs" TagsURI = "/tags" + StatsURI = "/stats" UserURI = "/user" RawPrefix = "/raw" diff --git a/pkg/core/types.go b/pkg/core/types.go index 7019d41..00f6dcf 100644 --- a/pkg/core/types.go +++ b/pkg/core/types.go @@ -64,6 +64,14 @@ func (p Paths) Len() int { return len(p) } func (p Paths) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p Paths) Less(i, j int) bool { return p[i].Name < p[j].Name } +type Stats struct { + Goroutines int + Alloc string + Sys string + Docs uint64 + GoVersion string +} + type Config struct { RunDir string `json:"run_dir,omitempty"` DataDir string `json:"-"` diff --git a/pkg/db/db.go b/pkg/db/db.go index b37b087..874001f 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -25,9 +25,7 @@ type DB struct { } func New(cfg core.Config) (db *DB, err error) { - db = new(DB) - dbFile := filepath.Join(cfg.RunDir, storeFile) - db.store, err = store.NewBoltStore(dbFile, nil) + db, err = NewPlain(cfg) if err != nil { return } @@ -50,6 +48,13 @@ func New(cfg core.Config) (db *DB, err error) { return } +func NewPlain(cfg core.Config) (db *DB, err error) { + db = new(DB) + dbFile := filepath.Join(cfg.RunDir, storeFile) + db.store, err = store.NewBoltStore(dbFile, nil) + return +} + func (db *DB) Dump(w io.Writer) error { return db.store.Dump(w) } @@ -63,5 +68,8 @@ func (db *DB) Close() (err error) { if err != nil { return } - return db.Index.Close() + if db.Index != nil { + return db.Index.Close() + } + return nil } diff --git a/pkg/fs/helper.go b/pkg/fs/helper.go index 6ae3315..5ee685c 100644 --- a/pkg/fs/helper.go +++ b/pkg/fs/helper.go @@ -27,6 +27,7 @@ var reserved = []string{ core.SearchURI, core.LogsURI, core.TagsURI, + core.StatsURI, core.UserURI, core.RawPrefix, core.IndexPrefix, diff --git a/pkg/index/index.go b/pkg/index/index.go index cedc7ba..fb8a5f0 100644 --- a/pkg/index/index.go +++ b/pkg/index/index.go @@ -66,6 +66,11 @@ func (i *Index) Close() error { return i.i.Close() } +func (i *Index) DocCount() (c uint64) { + c, _ = i.i.DocCount() + return +} + func (i *Index) Add(text string, tags []string) (id string, err error) { id = newID() for { diff --git a/pkg/server/handler.go b/pkg/server/handler.go index 0ae283b..8593913 100644 --- a/pkg/server/handler.go +++ b/pkg/server/handler.go @@ -174,51 +174,19 @@ func rawHandler(ctx *Context) { http.ServeContent(ctx.Response, ctx.Request, fi.Name(), fi.ModTime(), f) } -func scanFile(srv *HTTPServer, path string) (err error) { - err = srv.FS.AddScan(path) - if err != nil { - return - } - go func() { - defer srv.FS.RemoveScan(path) - l := srv.Log - file, txt, err := srv.Scanner.Scan(path) - if err != nil { - l.Printf("scanner: %s: %s", path, err) - return - } - tags, err := srv.DB.GetAllRTags() - if err != nil { - l.Printf("getTags: %s: %s", path, err) - } - found := tags.Match(txt) - id, err := srv.DB.Index.Add(txt, found) - if err != nil { - l.Printf("index: %s: %s", path, err) - return - } - err = srv.DB.NewFile(id, file, found) - if err != nil { - l.Printf("newFile: %s: %s", path, err) - } - }() - return -} - func indexHandler(ctx *Context) { switch ctx.Method() { case "GET": var ( err error + ok bool fpath string ) path := ctx.Path() - if strings.HasPrefix(path, core.IndexPrefix) { - fpath = strings.TrimPrefix(path, core.IndexPrefix) + if fpath, ok = hasTrimPrefix(path, core.IndexPrefix); ok { err = scanFile(ctx.Srv, fpath) } - if strings.HasPrefix(path, core.RetagPrefix) { - fpath = strings.TrimPrefix(path, core.RetagPrefix) + if fpath, ok = hasTrimPrefix(path, core.RetagPrefix); ok { if fpath == "" { go func() { err := ctx.Srv.DB.RetagAll() @@ -859,6 +827,19 @@ func tagsDelHandler(ctx *Context) { } } +func statsHandler(ctx *Context) { + ctx.Template("statsHandler") + ctx.Data = webData{ + Title: "Stats", + BodyTitle: "Stats", + } + switch ctx.Method() { + case "GET": + ctx.Data.Data = getStats(ctx.Srv.DB.Index) + ctx.Exec() + } +} + func logoutHandler(ctx *Context) { user := ctx.User() if user == "" { diff --git a/pkg/server/helper.go b/pkg/server/helper.go new file mode 100644 index 0000000..16cd2d6 --- /dev/null +++ b/pkg/server/helper.go @@ -0,0 +1,68 @@ +// Copyright (C) 2019 Marius Schellenberger + +package server + +import ( + "runtime" + "strconv" + + "git.giftfish.de/ston1th/docstore/pkg/core" + bleve "git.giftfish.de/ston1th/docstore/pkg/index" +) + +const div = 1048576 + +func fmtMem(i uint64) string { + return strconv.FormatFloat(float64(i)/div, 'f', 2, 64) +} + +func getStats(i *bleve.Index) *core.Stats { + mem := new(runtime.MemStats) + runtime.ReadMemStats(mem) + return &core.Stats{ + Goroutines: runtime.NumGoroutine(), + Alloc: fmtMem(mem.Alloc), + Sys: fmtMem(mem.Sys), + Docs: i.DocCount(), + GoVersion: runtime.Version(), + } +} + +func scanFile(srv *HTTPServer, path string) (err error) { + err = srv.FS.AddScan(path) + if err != nil { + return + } + go func() { + defer srv.FS.RemoveScan(path) + l := srv.Log + file, txt, err := srv.Scanner.Scan(path) + if err != nil { + l.Printf("scanner: %s: %s", path, err) + return + } + tags, err := srv.DB.GetAllRTags() + if err != nil { + l.Printf("getTags: %s: %s", path, err) + } + found := tags.Match(txt) + id, err := srv.DB.Index.Add(txt, found) + if err != nil { + l.Printf("index: %s: %s", path, err) + return + } + err = srv.DB.NewFile(id, file, found) + if err != nil { + l.Printf("newFile: %s: %s", path, err) + } + }() + return +} + +// Parts taken from strings.TrimPrefix +func hasTrimPrefix(s, prefix string) (string, bool) { + if len(s) >= len(prefix) && s[0:len(prefix)] == prefix { + return s[len(prefix):], true + } + return s, false +} diff --git a/pkg/server/routes.go b/pkg/server/routes.go index f610448..7084f8c 100644 --- a/pkg/server/routes.go +++ b/pkg/server/routes.go @@ -143,6 +143,13 @@ var routes = append(static, []route{ tagsHandler)), []string{"GET"}, }, + { + core.StatsURI, + jwtHandler( + authHandler( + statsHandler)), + []string{"GET"}, + }, { "/tags/new", jwtHandler( diff --git a/pkg/server/templates.go b/pkg/server/templates.go index 19cc578..46bed4a 100644 --- a/pkg/server/templates.go +++ b/pkg/server/templates.go @@ -325,6 +325,7 @@ const ( {{end}}