ston1th/docstore
1
1
Fork 0
Code Issues 3 Releases Activity

improved logging

Browse source
This commit is contained in:
ston1th 2022-09-19 22:52:40 +02:00
commit 05b41cedce
81 changed files with 9802 additions and 279 deletions
Download patch file Download diff file Expand all files Collapse all files

135
pkg/cmd/cmd.go
View file

@ -6,20 +6,22 @@ import (
"encoding/json"
"errors"
"fmt"
"git.giftfish.de/ston1th/docstore/pkg/core"
"git.giftfish.de/ston1th/docstore/pkg/db"
"git.giftfish.de/ston1th/docstore/pkg/log"
"git.giftfish.de/ston1th/docstore/pkg/scan"
"git.giftfish.de/ston1th/docstore/pkg/server"
"git.giftfish.de/ston1th/godrop/v2"
"github.com/urfave/cli"
stdlog "log"
"net"
"os"
"os/signal"
"path/filepath"
"strconv"
"syscall"
"time"
"git.giftfish.de/ston1th/docstore/pkg/core"
"git.giftfish.de/ston1th/docstore/pkg/db"
"git.giftfish.de/ston1th/docstore/pkg/scan"
"git.giftfish.de/ston1th/docstore/pkg/server"
"git.giftfish.de/ston1th/godrop/v2"
"github.com/go-logr/logr"
"github.com/go-logr/zerologr"
"github.com/rs/zerolog"
"github.com/urfave/cli"
)
const (
@ -72,15 +74,18 @@ func initServer(cfg *core.Config) (err error) {
return
}
log.InitLogger(cfg)
log, err := logFileLogger(cfg)
if err != nil {
return
}
cfg.DataDir = filepath.Join(cfg.RunDir, core.DataDir)
debugCfg := new(core.Config)
*debugCfg = *cfg
debugCfg.Secret = "***hidden***"
log.Debugf("docstore config: %+v", debugCfg)
debugCfg.Secret = "[redacted]"
log.V(2).Info("docstore config", "cfg", debugCfg)
scanner, err := scan.New(cfg)
scanner, err := scan.New(log.WithName("scanner"), cfg)
if err != nil {
return errors.New("scanner: " + err.Error())
}
@ -93,25 +98,21 @@ func initServer(cfg *core.Config) (err error) {
return
}
srv := server.NewHTTPServer(cfg, l, scanner)
srv := server.NewHTTPServer(log, cfg, l, scanner)
err = srv.Start()
if err != nil {
return
}
log.Println("docstore " + cfg.Version + " started")
log.Info("docstore started", "version", cfg.Version)
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
sig := <-sigs
log.Println("signal: " + sig.String())
go func() {
time.Sleep(time.Second * 20)
log.Fatal("stop timed out: killing")
}()
<-sigs
log.Info("docstore shutdown initiated")
err = srv.Stop()
if err != nil {
log.Println(err)
log.Error(err, "shutdown")
}
log.Println("docstore stopped")
log.Info("docstore shutdown completed")
return
}
@ -127,6 +128,9 @@ func loadConfig() error {
if err != nil {
return err
}
if config.Debug {
zerolog.SetGlobalLevel(zerolog.TraceLevel)
}
if config.RunDir == "" {
config.RunDir = defRunDir
}
@ -154,6 +158,47 @@ func loadConfig() error {
return file.Close()
}
func fatal(err error) {
log.Error(err, "init failed")
os.Exit(1)
}
var (
log logr.Logger
zl zerolog.Logger
)
func logFileLogger(cfg *core.Config) (l logr.Logger, err error) {
f, err := os.OpenFile(filepath.Join(cfg.DataDir, cfg.LogFile), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0o640)
if err != nil {
return
}
zln := zl.Output(f)
l = zerologr.New(&zln)
return
}
func init() {
//zerolog.TimeFieldFormat = zerolog.TimeFormatUnixMs
zerolog.CallerMarshalFunc = func(pc uintptr, file string, line int) string {
short := file
for i := len(file) - 1; i > 0; i-- {
if file[i] == '/' {
short = file[i+1:]
break
}
}
file = short
return file + ":" + strconv.Itoa(line)
}
//zerologr.NameFieldName = "logger"
//zerologr.NameSeparator = "/"
zerologr.VerbosityFieldName = ""
zl = zerolog.New(os.Stderr)
zl = zl.With().Caller().Timestamp().Logger()
log = zerologr.New(&zl)
}
func Run(version string) {
app := cli.NewApp()
app.Name = "docstore"
@ -167,11 +212,11 @@ func Run(version string) {
Action: func(c *cli.Context) error {
config.Langs = c.StringSlice("langs")
if err := loadConfig(); err != nil {
stdlog.Fatal("server: ", err)
fatal(err)
}
config.Version = app.Version
if err := initServer(&config); err != nil {
stdlog.Fatal("server: ", err)
fatal(err)
}
return nil
},
@ -182,30 +227,30 @@ func Run(version string) {
Flags: defaultFlags(nil),
Action: func(c *cli.Context) error {
if err := loadConfig(); err != nil {
stdlog.Fatal("reset: ", err)
fatal(err)
}
DB, err := db.NewPlain(&config)
DB, err := db.NewPlain(log.WithName("db"), &config)
if err != nil {
stdlog.Fatal("reset: ", err)
fatal(err)
}
u, err := DB.GetUser()
if err != nil {
stdlog.Fatal("reset: ", err)
fatal(err)
}
fmt.Println("username:", u.Username)
pw, err := password()
if err != nil {
stdlog.Fatal("reset: ", err)
fatal(err)
}
if err = DB.UpdateUserPassword(pw); err != nil {
stdlog.Fatal("reset: ", err)
fatal(err)
}
fmt.Println("password:", pw)
if err = DB.UpdateUserSecret(""); err != nil {
stdlog.Fatal("reset: ", err)
fatal(err)
}
if err = DB.Close(); err != nil {
stdlog.Fatal("reset: ", err)
fatal(err)
}
return nil
},
@ -220,25 +265,25 @@ func Run(version string) {
err error
)
if err = loadConfig(); err != nil {
stdlog.Fatal("dump: ", err)
fatal(err)
}
if dumpFile != "-" {
file, err = os.OpenFile(dumpFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o640)
if err != nil {
stdlog.Fatal("dump: ", err)
fatal(err)
}
}
DB, err := db.NewPlain(&config)
DB, err := db.NewPlain(log.WithName("db"), &config)
if err != nil {
stdlog.Fatal("dump: ", err)
fatal(err)
}
if err = DB.Dump(file); err != nil {
stdlog.Fatal("dump: ", err)
fatal(err)
}
file.Close()
err = DB.Close()
if err != nil {
stdlog.Fatal("dump: ", err)
fatal(err)
}
return nil
},
@ -253,25 +298,25 @@ func Run(version string) {
err error
)
if err = loadConfig(); err != nil {
stdlog.Fatal("restore: ", err)
fatal(err)
}
if dumpFile != "-" {
file, err = os.Open(dumpFile)
if err != nil {
stdlog.Fatal("restore: ", err)
fatal(err)
}
}
DB, err := db.NewPlain(&config)
DB, err := db.NewPlain(log.WithName("db"), &config)
if err != nil {
stdlog.Fatal("restore: ", err)
fatal(err)
}
if err = DB.Restore(file); err != nil {
stdlog.Fatal("restore: ", err)
fatal(err)
}
file.Close()
err = DB.Close()
if err != nil {
stdlog.Fatal("restore: ", err)
fatal(err)
}
return nil
},

15
pkg/db/db.go
View file

@ -3,11 +3,13 @@
package db
import (
"io"
"path/filepath"
"git.giftfish.de/ston1th/docstore/pkg/core"
"git.giftfish.de/ston1th/docstore/pkg/index"
"git.giftfish.de/ston1th/docstore/pkg/store"
"io"
"path/filepath"
"github.com/go-logr/logr"
)
const (
@ -18,14 +20,15 @@ const (
const blevePath = "bleve"
type DB struct {
log logr.Logger
rm countMutex
IndexMutex countMutex
store store.Store
Index *index.Index
}
func New(cfg *core.Config) (db *DB, err error) {
db, err = NewPlain(cfg)
func New(log logr.Logger, cfg *core.Config) (db *DB, err error) {
db, err = NewPlain(log, cfg)
if err != nil {
return
}
@ -48,8 +51,8 @@ func New(cfg *core.Config) (db *DB, err error) {
return
}
func NewPlain(cfg *core.Config) (db *DB, err error) {
db = new(DB)
func NewPlain(log logr.Logger, cfg *core.Config) (db *DB, err error) {
db = &DB{log: log}
dbFile := filepath.Join(cfg.RunDir, storeFile)
db.store, err = store.NewBoltStore(dbFile, nil)
return

8
pkg/db/migration.go
View file

@ -3,7 +3,6 @@
package db
import (
"git.giftfish.de/ston1th/docstore/pkg/log"
"git.giftfish.de/ston1th/docstore/pkg/store"
)
@ -32,6 +31,7 @@ var migrators = []migrator{
}
func (db *DB) runMigrations() (err error) {
log := db.log
var version int
err = db.store.Get(versionKey, &version)
if err == store.ErrKeyNotFound {
@ -44,10 +44,10 @@ func (db *DB) runMigrations() (err error) {
return
}
}
log.Printf("db: database version %d", version)
log.Info("database version", "version", version)
for version < currentVersion {
v := version + 1
log.Printf("db: running migration version %d", v)
log.Info("running migration", "version", v)
err = migrators[v](db, v)
if err != nil {
return
@ -57,7 +57,7 @@ func (db *DB) runMigrations() (err error) {
return
}
version = v
log.Printf("db: database version %d", version)
log.Info("database version", "version", version)
}
return
}

12
pkg/db/tags.go
View file

@ -5,10 +5,10 @@ package db
import (
"errors"
"fmt"
"git.giftfish.de/ston1th/docstore/pkg/core"
"git.giftfish.de/ston1th/docstore/pkg/log"
"regexp"
"sort"
"git.giftfish.de/ston1th/docstore/pkg/core"
)
const (
@ -34,6 +34,7 @@ func (db *DB) addDefaultTags() (err error) {
}
func (db *DB) Retag(path string, tags core.RTags) (err error) {
log := db.log.WithValues("file", path)
if tags == nil {
tags, err = db.GetAllRTags()
if err != nil {
@ -48,14 +49,14 @@ func (db *DB) Retag(path string, tags core.RTags) (err error) {
if err != nil {
return
}
log.Debugf("db: retagging: %s", path)
log.V(2).Info("retagging")
doc, err := db.Index.Get(id)
if err != nil {
return
}
doc.Tags = tags.Match(doc.Text)
if eq(t, doc.Tags) {
log.Debugf("db: retagging: skipped %s", path)
log.V(2).Info("retagging skipped")
return
}
err = db.Index.Update(id, doc)
@ -66,6 +67,7 @@ func (db *DB) Retag(path string, tags core.RTags) (err error) {
}
func (db *DB) RetagAll() (err error) {
log := db.log
if m, c, ok := db.rm.Lock(); !ok {
return fmt.Errorf("retagging already in progress: %d/%d", c, m)
}
@ -89,7 +91,7 @@ func (db *DB) RetagAll() (err error) {
if err == nil {
err = errors.New("retagAll had errors")
}
log.Printf("db: error retagging %s: %s", p, e)
log.Error(e, "error retagging file", "file", p)
}
db.rm.Inc()
}

9
pkg/fs/fs.go
View file

@ -4,13 +4,14 @@ package fs
import (
"errors"
"git.giftfish.de/ston1th/docstore/pkg/core"
"git.giftfish.de/ston1th/docstore/pkg/log"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"git.giftfish.de/ston1th/docstore/pkg/core"
"github.com/go-logr/logr"
)
const (
@ -32,7 +33,7 @@ type Filesystem struct {
scan map[string]struct{}
}
func NewFilesystem(base string) (fs *Filesystem, err error) {
func NewFilesystem(log logr.Logger, base string) (fs *Filesystem, err error) {
if !filepath.IsAbs(base) {
err = errors.New("base dir '" + base + "' is not an absolute path")
return
@ -43,7 +44,7 @@ func NewFilesystem(base string) (fs *Filesystem, err error) {
if err != nil {
return
}
log.Printf("fs: created data directory '%s'", base)
log.Info("created data directory", "dir", base)
fi, err = os.Stat(base)
if err != nil {
return

52
pkg/log/log.go
View file

@ -1,52 +0,0 @@
// Copyright (C) 2021 Marius Schellenberger
package log
import (
"git.giftfish.de/ston1th/docstore/pkg/core"
stdlog "log"
"os"
"path/filepath"
)
var (
log = stdlog.New(os.Stdout, "", stdlog.LstdFlags)
debug = false
)
func InitLogger(cfg *core.Config) {
debug = cfg.Debug
if cfg.LogFile == "-" {
return
}
f, err := os.OpenFile(filepath.Join(cfg.DataDir, cfg.LogFile), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0o640)
if err != nil {
stdlog.Fatal(err)
}
stdlog.SetOutput(f)
log = stdlog.New(f, "", stdlog.LstdFlags)
}
func Println(v ...interface{}) {
log.Println(v...)
}
func Printf(fmt string, v ...interface{}) {
log.Printf(fmt, v...)
}
func Fatal(v ...interface{}) {
log.Fatal(v...)
}
func Debug(v ...interface{}) {
if debug {
log.Println(append([]interface{}{"debug:"}, v...)...)
}
}
func Debugf(fmt string, v ...interface{}) {
if debug {
log.Printf("debug: "+fmt, v...)
}
}

72
pkg/log/scanlog.go
View file

@ -1,72 +0,0 @@
// Copyright (C) 2021 Marius Schellenberger
package log
import (
"fmt"
"git.giftfish.de/ston1th/docstore/pkg/core"
"strings"
"sync"
)
const defaultSize = 500
type ScanLog struct {
// protects logs
mu sync.RWMutex
size int
logs []string
}
func NewScanLog(size int) *ScanLog {
if size <= 0 {
size = defaultSize
}
return &ScanLog{size: size}
}
func (l *ScanLog) Printf(f string, v ...interface{}) {
s := fmt.Sprintf(f, v...)
Println(s)
s = core.LogTime() + s
l.mu.Lock()
llen := len(l.logs)
if llen < l.size {
l.logs = append(l.logs, s)
} else if llen == l.size {
l.logs = append(l.logs[1:], s)
}
l.mu.Unlock()
}
func (l *ScanLog) Logs() (logs string) {
l.mu.RLock()
logs = reverseJoin(l.logs, "\n")
l.mu.RUnlock()
return
}
// reverseJoin is a reverse version of strings.Join
// code taken from strings.Join
func reverseJoin(a []string, sep string) string {
switch len(a) {
case 0:
return ""
case 1:
return a[0]
}
n := len(sep) * (len(a) - 1)
for i := 0; i < len(a); i++ {
n += len(a[i])
}
var b strings.Builder
b.Grow(n)
b.WriteString(a[len(a)-1])
for i := len(a) - 2; i >= 0; i-- {
b.WriteString(sep)
b.WriteString(a[i])
}
return b.String()
}

18
pkg/scan/scanner.go
View file

@ -5,15 +5,16 @@ package scan
import (
"errors"
"fmt"
"git.giftfish.de/ston1th/docstore/pkg/core"
"git.giftfish.de/ston1th/docstore/pkg/fs"
"git.giftfish.de/ston1th/docstore/pkg/log"
"git.giftfish.de/ston1th/godrop/v2"
"os"
"os/exec"
"path/filepath"
"regexp"
"time"
"git.giftfish.de/ston1th/docstore/pkg/core"
"git.giftfish.de/ston1th/docstore/pkg/fs"
"git.giftfish.de/ston1th/godrop/v2"
"github.com/go-logr/logr"
)
const (
@ -44,13 +45,14 @@ var (
)
type Scanner struct {
log logr.Logger
base string
timeout time.Duration
t Tesseract
p PDF
}
func New(c *core.Config) (s *Scanner, err error) {
func New(log logr.Logger, c *core.Config) (s *Scanner, err error) {
var (
tcmd string
pcmd string
@ -92,6 +94,7 @@ func New(c *core.Config) (s *Scanner, err error) {
}
args := []string{"-l", lang, "--oem", fmt.Sprintf("%d", c.TesseractOEM)}
s = &Scanner{
log,
c.DataDir,
time.Second * time.Duration(c.CMDTimeout),
Tesseract{tcmd, args, []string{fmt.Sprintf("OMP_THREAD_LIMIT=%d", threads)}},
@ -101,6 +104,7 @@ func New(c *core.Config) (s *Scanner, err error) {
}
func (s *Scanner) Scan(file string) (filename, text string, err error) {
log := s.log
file = fs.Clean(file)
scanfile := filepath.Join(s.base, file)
var pdffile string
@ -117,7 +121,7 @@ func (s *Scanner) Scan(file string) (filename, text string, err error) {
p, _ := filepath.Split(file)
pdfCmd := exec.Command(s.t.cmd, append(s.t.args, []string{scanfile, "stdout", "pdf"}...)...)
pdfCmd.Env = s.t.env
log.Debugf("%v %s %v", pdfCmd.Env, pdfCmd.Path, pdfCmd.Args)
log.V(2).Info("tesseract command", "env", pdfCmd.Env, "path", pdfCmd.Path, "args", pdfCmd.Args)
pdf, err = timeout(pdfCmd, s.timeout)
if err != nil {
return
@ -140,7 +144,7 @@ func (s *Scanner) Scan(file string) (filename, text string, err error) {
filename = file
}
txtCmd := exec.Command(s.p.cmd, append(s.p.args, []string{pdffile, "-"}...)...)
log.Debugf("%s %v", txtCmd.Path, txtCmd.Args)
log.V(2).Info("text2pdf command", "path", txtCmd.Path, "args", txtCmd.Args)
txt, err := timeout(txtCmd, s.timeout)
if err != nil {
return

38
pkg/server/context.go
View file

@ -4,10 +4,6 @@ package server
import (
"errors"
"git.giftfish.de/ston1th/docstore/pkg/core"
"git.giftfish.de/ston1th/docstore/pkg/log"
"git.giftfish.de/ston1th/jwt/v3"
"github.com/gorilla/mux"
"html/template"
"io"
"net/http"
@ -15,6 +11,10 @@ import (
"reflect"
"strconv"
"time"
"git.giftfish.de/ston1th/docstore/pkg/core"
"git.giftfish.de/ston1th/jwt/v3"
"github.com/gorilla/mux"
)
const (
@ -93,7 +93,7 @@ func (c *Context) Exec() {
c.Err = errors.New("template is nil")
return
}
c.Data.Token = newXsrf(c.Token.RawSig()[:keySize])
c.Data.Token, c.Err = newXsrf(c.Token.RawSig()[:keySize])
c.Data.Version = c.Srv.Config.Version
if c.Data.BodyTitle == "" {
@ -123,10 +123,20 @@ func (c *Context) Log() {
c.HTTPStatus = int(reflect.Indirect(reflect.ValueOf(c.Response)).FieldByName("status").Int())
}
if c.Err != nil {
log.Printf("%s %s %s %d error: %s\n", c.Request.RemoteAddr, c.Request.Method, c.Request.URL, c.HTTPStatus, c.Err)
c.Srv.Log.Error(c.Err, "access",
"client", c.Request.RemoteAddr,
"method", c.Request.Method,
"status", c.HTTPStatus,
"uri", c.Request.URL.Path,
)
return
}
log.Printf("%s %s %s %d\n", c.Request.RemoteAddr, c.Request.Method, c.Request.URL, c.HTTPStatus)
c.Srv.Log.Info("access",
"client", c.Request.RemoteAddr,
"method", c.Request.Method,
"status", c.HTTPStatus,
"uri", c.Request.URL.Path,
)
}
func (c *Context) Error(i interface{}) {
@ -154,7 +164,7 @@ func (c *Context) Forbidden() {
}
func (c *Context) SwitchHandler() {
log.Debug("server: switching to normal handler")
c.Srv.Log.V(2).Info("switching to normal handler")
c.Srv.srv.Handler = c.Srv.handler
c.Redirect(core.IndexURI, http.StatusFound)
}
@ -181,7 +191,13 @@ func (c *Context) Redirect(uri string, code int) {
if len(uri) > 0 && uri[0] != '/' {
uri = "/" + uri
}
log.Printf("%s %s %s %d %s\n", c.Request.RemoteAddr, c.Request.Method, c.Request.URL, code, uri)
c.Srv.Log.Info("access",
"client", c.Request.RemoteAddr,
"method", c.Request.Method,
"status", code,
"uri", c.Request.URL.Path,
"redirect", uri,
)
http.Redirect(c.Response, c.Request, uri, code)
}
@ -244,7 +260,7 @@ func (c *Context) LoggedOn() (ok bool) {
}
func (c *Context) LogSetCookie(msg string, err error) {
log.Printf("ctx: %s %s\n", msg, err)
c.Srv.Log.Error(err, msg)
c.SetCookie(nil, 0)
}
@ -266,7 +282,7 @@ func (c *Context) setCookie(t *jwt.Token, d time.Duration) {
}
t.Claims[jwt.ExpClaim] = jwt.NewExp(d)
if err := c.Srv.JWT.Sign(t); err != nil {
log.Println(err)
c.Srv.Log.Error(err, "jwt sign")
return
}
c.Token = *t

86
pkg/server/handler.go
View file

@ -3,16 +3,16 @@
package server
import (
"git.giftfish.de/ston1th/docstore/pkg/core"
"git.giftfish.de/ston1th/docstore/pkg/fs"
"git.giftfish.de/ston1th/docstore/pkg/log"
"git.giftfish.de/ston1th/docstore/pkg/otp"
"git.giftfish.de/ston1th/jwt/v3"
"html/template"
"net/http"
"path/filepath"
"strings"
"time"
"git.giftfish.de/ston1th/docstore/pkg/core"
"git.giftfish.de/ston1th/docstore/pkg/fs"
"git.giftfish.de/ston1th/docstore/pkg/otp"
"git.giftfish.de/ston1th/jwt/v3"
)
const noSuchFile = "no such file or directory"
@ -146,10 +146,7 @@ func rawHandler(ctx *Context) {
f, fi, err := ctx.Srv.FS.GetFile(path)
if err != nil {
ctx.Status(http.StatusNotFound)
err = ctx.Write([]byte(noSuchFile))
if err != nil {
log.Println("file:", err)
}
ctx.Err = ctx.Write([]byte(noSuchFile))
return
}
ctx.Response.Header().Del("Content-Security-Policy")
@ -157,6 +154,7 @@ func rawHandler(ctx *Context) {
}
func indexHandler(ctx *Context) {
log := ctx.Srv.Log
switch ctx.Method() {
case "GET":
var (
@ -166,20 +164,28 @@ func indexHandler(ctx *Context) {
)
path := ctx.Path()
if fpath, ok = hasTrimPrefix(path, core.IndexPrefix); ok {
log = log.WithValues("file", fpath)
log.V(2).Info("started indexing file")
err = scanFile(ctx.Srv, fpath)
if err != nil {
ctx.Srv.Log.Printf("index: %s", err)
log.Error(err, "error indexing file")
}
} else if fpath, ok = hasTrimPrefix(path, core.RetagPrefix); ok {
if fpath == "" {
log.V(2).Info("started retagging all files")
go func() {
err := ctx.Srv.DB.RetagAll()
if err != nil {
ctx.Srv.Log.Printf("retagAll: %s", err)
log.Error(err, "error retagging all files")
}
}()
} else {
log = log.WithValues("file", fpath)
log.V(2).Info("started retagging file")
err = ctx.Srv.DB.Retag(fpath, nil)
if err != nil {
log.Error(err, "error retagging file")
}
}
}
p, _ := filepath.Split(fpath)
@ -201,6 +207,7 @@ func indexHandler(ctx *Context) {
return
}
if m, c, ok := ctx.Srv.DB.IndexMutex.Lock(); ok {
log.V(2).Info("started indexing all files")
go func() {
defer ctx.Srv.DB.IndexMutex.Unlock()
files := ctx.Srv.FS.RecursiveFiles("/")
@ -209,14 +216,14 @@ func indexHandler(ctx *Context) {
if !ctx.Srv.DB.IsIndexed(f) {
err := scanFile(ctx.Srv, f)
if err != nil {
ctx.Srv.Log.Printf("indexAll: %s", err)
log.Error(err, "error indexing file", "file", f)
}
}
ctx.Srv.DB.IndexMutex.Inc()
}
}()
} else {
ctx.Srv.Log.Printf("indexAll: indexing already in progress: %d/%d", c, m)
log.Info("indexing of all files is already in progress", "indexed", c, "remaning", m)
}
ctx.Redirect(core.IndexURI, http.StatusFound)
}
@ -276,6 +283,7 @@ func newDirHandler(ctx *Context) {
}
func moveHandler(ctx *Context) {
log := ctx.Srv.Log
path := strings.TrimPrefix(ctx.Path(), core.MovePrefix)
mode, _, err := ctx.Srv.FS.Mode(path)
if err != nil {
@ -320,18 +328,19 @@ func moveHandler(ctx *Context) {
return
}
for _, m := range moved {
log.V(2).Info("moving file", "old", m.Old, "new", m.New)
err = ctx.Srv.DB.MoveFile(m.Old, m.New)
if err != nil {
log.Printf("move: %s: %s", m, err)
log.Error(err, "moving file failed", "old", m.Old, "new", m.New)
continue
}
log.Debugf("move: %s to %s", m.Old, m.New)
}
ctx.Redirect(newfile, http.StatusFound)
}
}
func deleteHandler(ctx *Context) {
log := ctx.Srv.Log
path := strings.TrimPrefix(ctx.Path(), core.DeletePrefix)
mode, enoent, err := ctx.Srv.FS.Mode(path)
if !enoent && err != nil {
@ -374,18 +383,21 @@ func deleteHandler(ctx *Context) {
clean := ctx.Form("clean")
if clean == "true" {
f := fs.Clean(path)
log = log.WithValues("file", f)
log.V(2).Info("deleting file from DB")
id, _, err := ctx.Srv.DB.DeleteFile(f)
if err != nil {
log.Printf("db: delete: %s %s", f, err)
log.Error(err, "delete from DB failed")
}
if id == "" {
ctx.Redirect(p, http.StatusFound)
return
}
log.Debugf("delete %s id: %s from index", f, id)
log = log.WithValues("id", id)
log.V(2).Info("deleting file from index")
err = ctx.Srv.DB.Index.Delete(id)
if err != nil {
log.Printf("index: delete: %s %s", f, err)
log.Error(err, "delete from index failed")
}
ctx.Redirect(p, http.StatusFound)
return
@ -396,17 +408,18 @@ func deleteHandler(ctx *Context) {
return
}
for _, f := range files {
log.V(2).Info("deleting file from DB", "file", f)
id, _, err := ctx.Srv.DB.DeleteFile(f)
if err != nil {
log.Printf("db: delete: %s %s", f, err)
log.Error(err, "delete from DB failed", "file", f)
}
if id == "" {
continue
}
log.Debugf("delete %s id: %s from index", f, id)
log.V(2).Info("deleting file from index", "file", f, "id", id)
err = ctx.Srv.DB.Index.Delete(id)
if err != nil {
log.Printf("index: delete: %s %s", f, err)
log.Error(err, "delete from index failed", "file", f, "id", id)
}
}
ctx.Redirect(p, http.StatusFound)
@ -488,6 +501,7 @@ func loginHandler(ctx *Context) {
Title: "Login",
BodyTitle: "Login",
}
log := ctx.Srv.Log
var data loginData
switch ctx.Method() {
case "GET":
@ -511,12 +525,12 @@ func loginHandler(ctx *Context) {
}
u, err := ctx.Srv.DB.Login(data.User, password)
if err != nil {
log.Println("login:", data.User, err)
log.Error(err, "login failed", "user", data.User)
ctx.Error("bad username or password")
return
}
if u.Secret == "" {
log.Println("login:", u.Username)
log.Info("login successful", "user", u.Username)
ctx.Login(u, ctx.Form("remember"))
ctx.Redirect(data.Referer, http.StatusFound)
return
@ -540,6 +554,7 @@ func loginTotpHandler(ctx *Context) {
Title: "TOTP Verification",
BodyTitle: "TOTP Veriftcation",
}
log := ctx.Srv.Log
switch ctx.Method() {
case "GET":
ctx.Exec()
@ -558,7 +573,7 @@ func loginTotpHandler(ctx *Context) {
return
}
ref := ctx.Referer()
log.Println("login:", u.Username)
log.Info("login successful", "user", u.Username)
ctx.Login(u, ctx.Remember())
ctx.Redirect(ref, http.StatusFound)
}
@ -598,6 +613,7 @@ func userEditHandler(ctx *Context) {
Title: "Edit User",
BodyTitle: "Edit User",
}
log := ctx.Srv.Log
u, err := ctx.Srv.DB.GetUserWithoutPassword()
if err != nil {
ctx.Error(err)
@ -622,7 +638,7 @@ func userEditHandler(ctx *Context) {
ctx.Error(err)
return
}
log.Printf("update: %s updated\n", ctx.User())
log.Info("user updated", "user", ctx.User())
ctx.Redirect(core.IndexURI, http.StatusFound)
}
}
@ -633,6 +649,7 @@ func userTotpHandler(ctx *Context) {
Title: "TOTP",
BodyTitle: "TOTP",
}
log := ctx.Srv.Log
req := otp.Request{Username: ctx.User()}
u, err := ctx.Srv.DB.GetUser()
if err != nil {
@ -686,9 +703,9 @@ func userTotpHandler(ctx *Context) {
return
}
if req.Secret == "" {
log.Printf("totp: disabled for %s\n", user)
log.Info("totp disabled", "user", user)
} else {
log.Printf("totp: enabled for %s\n", user)
log.Info("totp enabled", "user", user)
}
ctx.Redirect("/user/edit", http.StatusFound)
}
@ -701,7 +718,8 @@ func logsHandler(ctx *Context) {
BodyTitle: "Logs",
}
if ctx.Method() == "GET" {
ctx.Data.Data = ctx.Srv.Log.Logs()
// TODO remove
//ctx.Data.Data = ctx.Srv.Log.Logs()
ctx.Exec()
}
}
@ -729,6 +747,7 @@ func tagsNewHandler(ctx *Context) {
Title: "New Tag",
BodyTitle: "New Tag",
}
log := ctx.Srv.Log
switch ctx.Method() {
case "GET":
ctx.Data.Data = core.Tag{}
@ -753,7 +772,7 @@ func tagsNewHandler(ctx *Context) {
ctx.Error(err)
return
}
log.Printf("tag %s created by: %s\n", t.Name, ctx.User())
log.Info("new tag created", "user", ctx.User(), "tag", t.Name)
ctx.Redirect(core.TagsURI, http.StatusFound)
}
}
@ -764,6 +783,7 @@ func tagsEditHandler(ctx *Context) {
Title: "Edit Tag",
BodyTitle: "Edit Tag",
}
log := ctx.Srv.Log
name := ctx.Var("tag")
switch ctx.Method() {
case "GET":
@ -790,7 +810,7 @@ func tagsEditHandler(ctx *Context) {
ctx.Error(err)
return
}
log.Printf("tag %s updated by: %s\n", t.Name, ctx.User())
log.Info("tag updated", "user", ctx.User(), "tag", t.Name)
ctx.Redirect(core.TagsURI, http.StatusFound)
}
}
@ -801,6 +821,7 @@ func tagsDelHandler(ctx *Context) {
Title: "Delete Tag",
BodyTitle: "Delete Tag",
}
log := ctx.Srv.Log
name := ctx.Var("tag")
ctx.Data.Data = name
switch ctx.Method() {
@ -814,7 +835,7 @@ func tagsDelHandler(ctx *Context) {
ctx.Error(err)
return
}
log.Printf("tag %s deleted by: %s\n", name, ctx.User())
log.Info("tag deleted", "user", ctx.User(), "tag", name)
ctx.Redirect(core.TagsURI, http.StatusFound)
}
}
@ -832,11 +853,12 @@ func statsHandler(ctx *Context) {
}
func logoutHandler(ctx *Context) {
log := ctx.Srv.Log
user := ctx.User()
if user == "" {
user = ctx.Totp() + " (totp)"
}
log.Println("logout:", user)
log.Info("user logged out", "user", user)
ctx.Srv.JWT.Invalidate(&ctx.Token)
ctx.SetCookie(nil, 0)
ctx.Redirect(core.LoginURI, http.StatusFound)

10
pkg/server/helper.go
View file

@ -35,25 +35,25 @@ func scanFile(srv *HTTPServer, path string) (err error) {
}
go func() {
defer srv.FS.RemoveScan(path)
l := srv.Log
log := srv.Log.WithValues("file", path)
file, txt, err := srv.Scanner.Scan(path)
if err != nil {
l.Printf("scanner: %s: %s", path, err)
log.Error(err, "error scanning file")
return
}
tags, err := srv.DB.GetAllRTags()
if err != nil {
l.Printf("getTags: %s: %s", path, err)
log.Error(err, "error getting tags")
}
found := tags.Match(txt)
id, err := srv.DB.Index.Add(txt, found)
if err != nil {
l.Printf("index: %s: %s", path, err)
log.Error(err, "error adding file to index")
return
}
err = srv.DB.NewFile(id, file, found)
if err != nil {
l.Printf("newFile: %s: %s", path, err)
log.Error(err, "error adding file to DB")
}
}()
return

50
pkg/server/server.go
View file

@ -7,20 +7,21 @@ import (
"context"
"embed"
"errors"
"git.giftfish.de/ston1th/authdav"
"git.giftfish.de/ston1th/docstore/pkg/core"
"git.giftfish.de/ston1th/docstore/pkg/db"
"git.giftfish.de/ston1th/docstore/pkg/fs"
"git.giftfish.de/ston1th/docstore/pkg/log"
"git.giftfish.de/ston1th/docstore/pkg/scan"
"git.giftfish.de/ston1th/godrop/v2"
"git.giftfish.de/ston1th/jwt/v3"
"github.com/gorilla/mux"
"golang.org/x/net/webdav"
"html/template"
"net"
"net/http"
"time"
"git.giftfish.de/ston1th/authdav"
"git.giftfish.de/ston1th/docstore/pkg/core"
"git.giftfish.de/ston1th/docstore/pkg/db"
"git.giftfish.de/ston1th/docstore/pkg/fs"
"git.giftfish.de/ston1th/docstore/pkg/scan"
"git.giftfish.de/ston1th/godrop/v2"
"git.giftfish.de/ston1th/jwt/v3"
"github.com/go-logr/logr"
"github.com/gorilla/mux"
"golang.org/x/net/webdav"
)
const (
@ -38,18 +39,20 @@ type HTTPServer struct {
JWT *jwt.JWT
Scanner *scan.Scanner
FS *fs.Filesystem
Log *log.ScanLog
Log logr.Logger
plain logr.Logger
templ map[string]*template.Template
res map[string][]byte
}
func NewHTTPServer(cfg *core.Config, l net.Listener, s *scan.Scanner) (srv *HTTPServer) {
func NewHTTPServer(log logr.Logger, cfg *core.Config, l net.Listener, s *scan.Scanner) (srv *HTTPServer) {
srv = &HTTPServer{
Config: cfg,
listener: l,
Scanner: s,
Log: log.NewScanLog(0),
Log: log.WithName("server"),
plain: log,
}
srv.loadTemplates()
srv.srv = &http.Server{}
@ -57,11 +60,12 @@ func NewHTTPServer(cfg *core.Config, l net.Listener, s *scan.Scanner) (srv *HTTP
}
func (s *HTTPServer) Start() (err error) {
s.FS, err = fs.NewFilesystem(s.Config.DataDir)
log := s.Log
s.FS, err = fs.NewFilesystem(s.plain.WithName("fs"), s.Config.DataDir)
if err != nil {
return errors.New("fs: " + err.Error())
}
s.DB, err = db.New(s.Config)
s.DB, err = db.New(s.plain.WithName("db"), s.Config)
if err != nil {
return errors.New("db: " + err.Error())
}
@ -79,13 +83,16 @@ func (s *HTTPServer) Start() (err error) {
}
s.handler = s.buildRoutes()
if !s.DB.UserExists() {
log.Debug("server: starting register handler")
log.V(2).Info("starting register handler")
s.srv.Handler = s.register()
} else {
s.srv.Handler = s.handler
}
go func() {
log.Println(s.srv.Serve(s.listener))
err = s.srv.Serve(s.listener)
if err != nil {
log.Error(err, "http server error")
}
}()
return
}
@ -117,7 +124,14 @@ func (s *HTTPServer) buildRoutes() http.Handler {
if s.Config.WebDav {
dav := authdav.NewWriteOnlyOnceFileSystem(webdav.Dir(s.Config.DataDir))
dav.Filters = []authdav.Filter{authdav.NewMacOSFilter()}
h := authdav.NewWebdavBasicAuth(core.WebDavPrefix, dav, nil, webdavLogger, s.DB, "DocStore WebDav")
h := authdav.NewWebdavBasicAuth(
core.WebDavPrefix,
dav,
nil,
webdavLogger(s.plain.WithName("webdav")),
s.DB,
"DocStore WebDav",
)
r.PathPrefix(core.WebDavPrefix).Handler(h)
}
r.Handle("/static/{file}", http.StripPrefix("/", http.FileServer(http.FS(static))))

10
pkg/server/templates.go
View file

@ -5,7 +5,6 @@ package server
import (
"embed"
"errors"
"git.giftfish.de/ston1th/docstore/pkg/log"
"html/template"
"io/fs"
)
@ -13,19 +12,19 @@ import (
//go:embed templates/*
var templates embed.FS
func (s *HTTPServer) loadTemplates() {
func (s *HTTPServer) loadTemplates() error {
var fatal bool
s.templ = make(map[string]*template.Template)
s.res = make(map[string][]byte)
tfs, err := fs.Sub(templates, "templates")
if err != nil {
log.Fatal("parse: ", err)
return err
}
parse := func(html ...string) (temp *template.Template) {
var err error
temp, err = template.ParseFS(tfs, html...)
if err != nil {
log.Println("parse:", err)
s.Log.Error(err, "template parser")
fatal = true
}
return
@ -61,6 +60,7 @@ func (s *HTTPServer) loadTemplates() {
s.templ["logsHandler"] = parse("index.html", "menu.html", "logs.html")
if fatal {
log.Fatal("parse: ", errors.New("parsing templates failed"))
return errors.New("parsing templates failed")
}
return nil
}

25
pkg/server/webdav.go
View file

@ -1,16 +1,27 @@
// Copyright (C) 2021 Marius Schellenberger
// Copyright (C) 2022 Marius Schellenberger
package server
import (
"git.giftfish.de/ston1th/docstore/pkg/log"
"net/http"
"github.com/go-logr/logr"
)
func webdavLogger(r *http.Request, err error) {
if err != nil {
log.Printf("webdav: %s %s %s error: %s\n", r.RemoteAddr, r.Method, r.URL, err)
return
func webdavLogger(log logr.Logger) func(r *http.Request, err error) {
return func(r *http.Request, err error) {
if err != nil {
log.Error(err, "access",
"client", r.RemoteAddr,
"method", r.Method,
"uri", r.URL.Path,
)
return
}
log.Info("access",
"client", r.RemoteAddr,
"method", r.Method,
"uri", r.URL.Path,
)
}
log.Printf("webdav: %s %s %s\n", r.RemoteAddr, r.Method, r.URL)
}

17
pkg/server/xsrf.go
View file

@ -6,23 +6,22 @@ import (
"crypto/rand"
"crypto/subtle"
"encoding/base64"
"git.giftfish.de/ston1th/docstore/pkg/log"
"git.giftfish.de/ston1th/jwt/v3"
)
const keySize = jwt.KeySize / 2
func genKey(length int) (bytes []byte) {
bytes = make([]byte, length)
if _, err := rand.Reader.Read(bytes); err != nil {
log.Println("genKey:", err)
}
func genKey(size int) (b []byte, err error) {
b = make([]byte, size)
_, err = rand.Reader.Read(b)
return
}
func newXsrf(secret []byte) string {
rnd := genKey(keySize)
return base64.RawURLEncoding.EncodeToString(append(rnd, xor(rnd, secret)...))
func newXsrf(secret []byte) (s string, err error) {
rnd, err := genKey(keySize)
s = base64.RawURLEncoding.EncodeToString(append(rnd, xor(rnd, secret)...))
return
}
func checkXsrf(xsrf string, secret []byte) bool {

6
pkg/server/xsrf_test.go
View file

@ -7,11 +7,11 @@ import (
)
func TestXsrf(t *testing.T) {
sec := genKey(keySize)
sec, _ := genKey(keySize)
if len(sec) != keySize {
t.Fatal("len(sec) != keySize")
}
xsrf := newXsrf(sec)
xsrf, _ := newXsrf(sec)
if xsrf == "" {
t.Fatal("xsrf is empty")
}
@ -24,7 +24,7 @@ func TestXsrf(t *testing.T) {
t.Fatal("invalid xsrf check succeeded")
}
rnd := genKey(keySize)
rnd, _ := genKey(keySize)
if checkXsrf(xsrf, rnd) {
t.Fatal("invalid xsrf check succeeded")
}