finished tag implementation

This commit is contained in:
ston1th 2019-08-24 14:27:58 +02:00
commit c3076c328a
28 changed files with 396 additions and 296 deletions

View file

@ -253,7 +253,7 @@ func serverFlags() []cli.Flag {
},
cli.StringSliceFlag{
Name: "langs",
Usage: "languages for tesseract ocr",
Usage: "languages for tesseract ocr (use --langs multiple times to add more languages)",
},
cli.IntFlag{
Name: "timeout, t",

View file

@ -8,6 +8,7 @@ const (
LoginURI = "/login"
LogoutURI = "/logout"
SearchURI = "/search"
LogsURI = "/logs"
TagsURI = "/tags"
UserURI = "/user"

View file

@ -9,6 +9,7 @@ import (
const (
fileFmt = "20060102_150405"
logFmt = "2006/01/02 15:04:05: "
)
var m sync.Mutex
@ -21,3 +22,7 @@ func FileName() string {
}()
return time.Now().Format(fileFmt)
}
func LogTime() string {
return time.Now().Format(logFmt)
}

View file

@ -30,6 +30,7 @@ type Path struct {
Abs string
Flag bool
Scan bool
Tags []string
}
type Paths []Path

View file

@ -11,19 +11,19 @@ const (
pathPrefix = "path/"
)
func (db *DB) GetPaths(res []core.Result) error {
func (db *DB) GetPaths(res []core.Result) {
for i, v := range res {
err := db.store.Get(pathPrefix+v.ID, &res[i].Path)
if err != nil {
return err
res[i].Path = err.Error()
}
}
return nil
}
func (db *DB) GetIndexed(p core.Paths) {
for i, v := range p {
p[i].Flag = db.IsIndexed(v.Abs)
p[i].Tags, _ = db.GetTag(v.Abs)
}
}
@ -65,22 +65,14 @@ func (db *DB) MoveFile(op, np string) (err error) {
if !db.IsIndexed(op) {
return nil
}
id, err := db.DeleteFile(op)
id, tags, err := db.DeleteFile(op)
if err != nil {
return
}
tags, err := db.GetTag(op)
if err != nil {
return
}
err = db.NewFile(id, np, tags)
if err != nil {
return
}
return db.UnsetTag(op)
return db.NewFile(id, np, tags)
}
func (db *DB) DeleteFile(path string) (id string, err error) {
func (db *DB) DeleteFile(path string) (id string, tags []string, err error) {
if !db.IsIndexed(path) {
return
}
@ -92,7 +84,7 @@ func (db *DB) DeleteFile(path string) (id string, err error) {
if err != nil {
return
}
err = db.UnsetTag(path)
tags, err = db.UnsetTag(path)
if err != nil {
return
}

View file

@ -41,6 +41,11 @@ func (db *DB) GetAllTags() (tags []core.Tag, err error) {
return
}
func (db *DB) GetTags(name string) (t core.Tag, err error) {
err = db.store.Get(tagsPrefix+name, &t)
return
}
func (db *DB) DeleteTag(name string) error {
return db.store.Delete(tagsPrefix + name)
}
@ -64,6 +69,11 @@ func (db *DB) SetTag(path string, tags []string) error {
return db.store.Set(tagPrefix+path, tags)
}
func (db *DB) UnsetTag(path string) error {
return db.store.Delete(tagPrefix + path)
func (db *DB) UnsetTag(path string) (tags []string, err error) {
tags, err = db.GetTag(path)
if err != nil {
return
}
err = db.store.Delete(tagPrefix + path)
return
}

View file

@ -26,6 +26,7 @@ type Directory struct {
type Filesystem struct {
Base string
Log *log.ScanLog
// protects scan
m sync.RWMutex
@ -53,7 +54,7 @@ func NewFilesystem(base string) (fs *Filesystem, err error) {
err = errors.New("base dir '" + base + "' is not a directory")
return
}
fs = &Filesystem{Base: base, scan: make(map[string]struct{})}
fs = &Filesystem{Base: base, Log: log.NewScanLog(0), scan: make(map[string]struct{})}
return
}

View file

@ -25,6 +25,7 @@ var reserved = []string{
core.LoginURI,
core.LogoutURI,
core.SearchURI,
core.LogsURI,
core.TagsURI,
core.UserURI,
core.RawPrefix,

47
pkg/log/scanlog.go Normal file
View file

@ -0,0 +1,47 @@
package log
import (
"fmt"
"git.giftfish.de/ston1th/docstore/pkg/core"
"sync"
)
const defaultSize = 500
type ScanLog struct {
// protects logs
sync.RWMutex
size int
logs []string
}
func NewScanLog(size int) *ScanLog {
if size <= 0 {
size = defaultSize
}
return &ScanLog{size: size}
}
func (l *ScanLog) Printf(f string, v ...interface{}) {
s := fmt.Sprintf(f, v...)
Println(s)
s = core.LogTime() + s
l.Lock()
logsLen := len(l.logs)
if logsLen < l.size {
l.logs = append(l.logs, s)
} else if logsLen == l.size {
l.logs = append(l.logs[1:], s)
}
l.Unlock()
}
func (l *ScanLog) Logs() (logs string) {
l.RLock()
for _, v := range l.logs {
logs += "\n" + v
}
l.RUnlock()
return
}

View file

@ -3,10 +3,35 @@
package scan
import (
"net/http"
"os"
"path/filepath"
"strings"
)
const (
pdfContentType = "application/pdf"
maxSniffSize = 512
)
func isPDF(path string) bool {
return checkContentType(path) || checkExtension(path)
}
func checkContentType(path string) bool {
f, err := os.Open(path)
if err != nil {
return false
}
defer f.Close()
buf := make([]byte, maxSniffSize)
_, err = f.Read(buf)
if err != nil {
return false
}
return http.DetectContentType(buf) == pdfContentType
}
func checkExtension(path string) bool {
return strings.ToLower(filepath.Ext(path)) == pdfExt
}

View file

@ -76,6 +76,7 @@ func New(c core.Config) (s *Scanner, err error) {
langs := c.Langs
if langs == nil || llen == 0 {
langs = defaultLangs
llen = len(langs)
}
for i, l := range langs {
lang += l

View file

@ -174,49 +174,45 @@ func rawHandler(ctx *Context) {
http.ServeContent(ctx.Response, ctx.Request, fi.Name(), fi.ModTime(), f)
}
func scanFile(ctx *Context, path string) (err error) {
err = ctx.Srv.FS.AddScan(path)
if err != nil {
return
}
go func() {
defer ctx.Srv.FS.RemoveScan(path)
l := ctx.Srv.FS.Log
file, txt, err := ctx.Srv.Scanner.Scan(path)
if err != nil {
l.Printf("scanner: %s: %s", path, err)
return
}
t, err := ctx.Srv.DB.GetAllTags()
if err != nil {
l.Printf("getTags: %s: %s", path, err)
}
found := tags.Find(txt, t)
id, err := ctx.Srv.DB.Index.Add(txt, found)
if err != nil {
l.Printf("index: %s: %s", path, err)
return
}
err = ctx.Srv.DB.NewFile(id, file, found)
if err != nil {
l.Printf("newFile: %s: %s", path, err)
}
}()
return
}
func indexHandler(ctx *Context) {
path := strings.TrimPrefix(ctx.Path(), core.IndexPrefix)
switch ctx.Method() {
case "GET":
earlyErr := make(chan error)
go func() {
err := ctx.Srv.FS.AddScan(path)
if err != nil {
select {
case earlyErr <- err:
default:
}
return
}
defer ctx.Srv.FS.RemoveScan(path)
file, txt, err := ctx.Srv.Scanner.Scan(path)
if err != nil {
select {
case earlyErr <- err:
default:
}
log.Printf("scanner: %s: %s", path, err)
return
}
t, err := ctx.Srv.DB.GetAllTags()
if err != nil {
log.Printf("getTags: %s: %s", path, err)
}
found := tags.Find(txt, t)
id, err := ctx.Srv.DB.Index.Add(txt, found)
if err != nil {
log.Printf("index: %s: %s", path, err)
return
}
err = ctx.Srv.DB.NewFile(id, file, found)
if err != nil {
log.Printf("newFile: %s: %s", path, err)
}
}()
err := scanFile(ctx, path)
p, _ := filepath.Split(path)
p = filepath.Dir(p)
select {
case err := <-earlyErr:
if err != nil {
ctx.Template("iseHandler")
ctx.Data = webData{
Data: template.HTML(strings.ReplaceAll(err.Error(), "\n", "<br>")),
@ -225,7 +221,6 @@ func indexHandler(ctx *Context) {
ctx.Status(http.StatusInternalServerError)
ctx.Exec()
return
case <-time.After(time.Second):
}
ctx.Redirect(p, http.StatusFound)
}
@ -383,7 +378,7 @@ func deleteHandler(ctx *Context) {
clean := ctx.Form("clean")
if clean == "true" {
f := fs.Clean(path)
id, err := ctx.Srv.DB.DeleteFile(f)
id, _, err := ctx.Srv.DB.DeleteFile(f)
if err != nil {
log.Printf("db: delete: %s %s", f, err)
}
@ -405,7 +400,7 @@ func deleteHandler(ctx *Context) {
return
}
for _, f := range files {
id, err := ctx.Srv.DB.DeleteFile(f)
id, _, err := ctx.Srv.DB.DeleteFile(f)
if err != nil {
log.Printf("db: delete: %s %s", f, err)
}
@ -581,11 +576,7 @@ func searchHandler(ctx *Context) {
ctx.Error(err)
return
}
err = ctx.Srv.DB.GetPaths(res)
if err != nil {
ctx.Error(err)
return
}
ctx.Srv.DB.GetPaths(res)
ctx.Data.Data = res
ctx.Exec()
}
@ -693,71 +684,117 @@ func userTotpHandler(ctx *Context) {
}
}
func logsHandler(ctx *Context) {
ctx.Template("logsHandler")
ctx.Data = webData{
Title: "Scan Logs",
BodyTitle: "Scan Logs",
}
switch ctx.Method() {
case "GET":
ctx.Data.Data = ctx.Srv.FS.Log.Logs()
ctx.Exec()
}
}
func tagsHandler(ctx *Context) {
ctx.Template("tagsHandler")
ctx.Data = webData{
Title: "Tags",
BodyTitle: "Tags",
}
t, err := ctx.Srv.DB.GetAllTags()
if err != nil {
ctx.Error(err)
return
}
ctx.Data.Data = t
switch ctx.Method() {
case "GET":
t, err := ctx.Srv.DB.GetAllTags()
if err != nil {
ctx.Error(err)
return
}
ctx.Data.Data = t
ctx.Exec()
}
}
func tagsNewHandler(ctx *Context) {
ctx.Template("tagsNewHandler")
ctx.Data = webData{
Title: "New Tag",
BodyTitle: "New Tag",
}
switch ctx.Method() {
case "GET":
ctx.Data.Data = core.Tag{}
ctx.Exec()
case "POST":
if !ctx.CheckXsrf() {
return
}
var t core.Tag
t.Name = ctx.Form("name")
t.Regex = ctx.Form("regex")
ctx.Data.Data = t
if t.Name == "" {
ctx.Error("name cannot be empty")
return
}
if t.Regex == "" {
ctx.Error("regex cannot be empty")
return
}
if err := ctx.Srv.DB.NewTag(t.Name, t.Regex); err != nil {
ctx.Error(err)
return
}
log.Printf("tag %s created by: %s\n", t.Name, ctx.User())
ctx.Redirect(core.TagsURI, http.StatusFound)
}
}
func tagsEditHandler(ctx *Context) {
ctx.Template("tagsEditHandler")
ctx.Data = webData{
Title: "Edit User",
BodyTitle: "Edit User",
Title: "Edit Tag",
BodyTitle: "Edit Tag",
}
u, err := ctx.Srv.DB.GetUserWithoutPassword()
if err != nil {
ctx.Error(err)
return
}
ctx.Data.Data = u
name := ctx.Var("tag")
switch ctx.Method() {
case "GET":
t, err := ctx.Srv.DB.GetTags(name)
if err != nil {
ctx.Error(err)
return
}
ctx.Data.Data = t
ctx.Exec()
case "POST":
if !ctx.CheckXsrf() {
return
}
password := ctx.Form("password")
repeat := ctx.Form("repeat")
if password != repeat {
ctx.Error("passwords do not match")
var t core.Tag
t.Name = name
t.Regex = ctx.Form("regex")
ctx.Data.Data = t
if t.Regex == "" {
ctx.Error("regex cannot be empty")
return
}
if err := ctx.Srv.DB.UpdateUserPassword(password); err != nil {
if err := ctx.Srv.DB.NewTag(t.Name, t.Regex); err != nil {
ctx.Error(err)
return
}
log.Printf("update: %s updated\n", ctx.User())
ctx.Redirect(core.IndexURI, http.StatusFound)
log.Printf("tag %s updated by: %s\n", t.Name, ctx.User())
ctx.Redirect(core.TagsURI, http.StatusFound)
}
}
func tagsDelHandler(ctx *Context) {
ctx.Template("tagsDelHandler")
ctx.Data = webData{
Title: "Edit User",
BodyTitle: "Edit User",
Title: "Delete Tag",
BodyTitle: "Delete Tag",
}
u, err := ctx.Srv.DB.GetUserWithoutPassword()
if err != nil {
ctx.Error(err)
return
}
ctx.Data.Data = u
name := ctx.Var("tag")
ctx.Data.Data = name
switch ctx.Method() {
case "GET":
ctx.Exec()
@ -765,19 +802,12 @@ func tagsDelHandler(ctx *Context) {
if !ctx.CheckXsrf() {
return
}
password := ctx.Form("password")
repeat := ctx.Form("repeat")
if password != repeat {
ctx.Error("passwords do not match")
return
}
if err := ctx.Srv.DB.UpdateUserPassword(password); err != nil {
if err := ctx.Srv.DB.DeleteTag(name); err != nil {
ctx.Error(err)
return
}
log.Printf("update: %s updated\n", ctx.User())
ctx.Redirect(core.IndexURI, http.StatusFound)
log.Printf("tag %s deleted by: %s\n", name, ctx.User())
ctx.Redirect(core.TagsURI, http.StatusFound)
}
}

View file

@ -123,21 +123,35 @@ var routes = append(static, []route{
[]string{"GET", "POST"},
},
{
"/tags",
core.LogsURI,
jwtHandler(
authHandler(
logsHandler)),
[]string{"GET"},
},
{
core.TagsURI,
jwtHandler(
authHandler(
tagsHandler)),
[]string{"GET"},
},
{
"/tags/{tag:[a-zA-Z0-9]+}/edit",
"/tags/new",
jwtHandler(
authHandler(
tagsNewHandler)),
[]string{"GET", "POST"},
},
{
"/tags/edit/{tag:[a-zA-Z0-9]+}",
jwtHandler(
authHandler(
tagsEditHandler)),
[]string{"GET", "POST"},
},
{
"/tags/{tag:[a-zA-Z0-9]+}/del",
"/tags/del/{tag:[a-zA-Z0-9]+}",
jwtHandler(
authHandler(
tagsDelHandler)),

View file

@ -60,6 +60,7 @@ const (
<tr>
<th>Path</th>
<th>Type</th>
<th>Tags</th>
<th>Options</th>
</tr>
</thead>
@ -69,6 +70,7 @@ const (
<tr>
<th><a href="{{$item.Abs}}">{{$item.Name}}</a></th>
<td>Directory</td>
<td></td>
<td><div class="btn-group">
<a href="/move{{$item.Abs}}" class="btn btn-sm btn-primary">Move</a>
<a href="/delete{{$item.Abs}}" class="btn btn-sm btn-danger">Delete</a>
@ -79,6 +81,11 @@ const (
<tr>
<th><a href="{{$item.Abs}}">{{$item.Name}}</a></th>
<td>File</td>
<td>
{{range $t := $item.Tags}}
<code>{{$t}}</code>
{{end}}
</td>
<td><div class="btn-group">
{{if $item.Scan}}
<a href="#" class="btn btn-sm btn-warning">Scanning</a>
@ -160,7 +167,7 @@ const (
<title>DocStore | {{.Title}}</title>
<link rel="shortcut icon" type="image/x-icon" href="/favicon.ico" integrity="sha256-bgotk/IPq4Yvt6s8AQGPTM5ZjPjQ6rrBRa2EyxpnFSc="></link>
<link rel="stylesheet" type="text/css" href="/bootstrap.css" media="screen" integrity="sha256-3YeveuySvBgp2GItUS8eV1R/9T1zcPMyvw22SyVf8qo="></link>
<link rel="stylesheet" type="text/css" href="/custom.css" media="screen" integrity="sha256-NLGEBzpF8t2UDEPzWyOYGGP/HWLNb29ar1MdYlwJroQ="></link>
<link rel="stylesheet" type="text/css" href="/custom.css" media="screen" integrity="sha256-TjpqFXPvQzoakyE489hpNO1YK+wUAVd/AdwqpNz1gwc="></link>
<meta name="theme-color" content="#375a7f">
<meta name="description" content="{{.Title}}">
</head>
@ -262,19 +269,32 @@ const (
</div>
</div>
</div>
{{end}}`
logs = `{{define "body"}}
<div class="page-header">
<div class="row">
<h2>{{.BodyTitle}}</h2>
</div>
</div>
<div class="row mt-3">
<textarea class="form-control input-sm md-text mono" rows="30" spellcheck="false" autofocus>{{.Data}}</textarea>
</div>
{{end}}`
menu = `{{define "menu"}}
<ul class="navbar-nav">
<li class="nav-item"><a class="nav-link" href="/">Root</a></li>
{{if .Login}}
<li class="nav-item"><a class="nav-link" href="/tags">Tags</a></li>
<li class="nav-item"><a class="nav-link" href="/logs">Logs</a></li>
{{end}}
</ul>
{{if .Login}}
<form class="form-inline" method="post" action="/search">
<input type="hidden" name="token" value="{{.Token}}">
<input type="text" class="form-control mr-sm-2 menu" placeholder="Search" name="search" value="{{.Search}}" autocomplete="off">
<button class="btn btn-sm btn-info" type="submit">Search</button>
</form>
{{end}}
<ul class="nav navbar-nav ml-auto">
{{if .Login}}
<li class="nav-item"><a class="nav-link" href="/user/edit">Profile</a></li>
@ -327,9 +347,10 @@ const (
<input type="hidden" name="token" value="{{.Token}}">
<div class="form-group">
<label class="col-form-label" for="name">Name</label>
<input class="form-control input-sm" type="text" id="name" name="name" autocomplete="off" value="{{.Data}}">
<input class="form-control input-sm" type="text" id="name" name="name" autocomplete="off" value="{{.Data}}" autofocus required>
</div>
<button class="btn btn-sm btn-primary" type="submit">Create</button>
<a href="/" class="btn btn-sm btn-primary">Back</a>
</form>
</div>
</div>
@ -409,7 +430,7 @@ const (
<thead>
<tr>
<th>File</th>
<th>Keyword</th>
<th>Tags</th>
</tr>
</thead>
<tbody>
@ -421,7 +442,7 @@ const (
<br><pre class="wrap">{{$item.HTML}}</pre>
{{end}}
</td>
<td>{{$item.Keyword}}</td>
<td>{{$item.Tags}}</td>
</tr>
{{end}}
{{else}}
@ -439,17 +460,11 @@ const (
<strong>{{.BodyTitle}}</strong>
</div>
<div class="card-body">
<form class="form-horizontal" action="/tags/{{.Data.Name}}/edit" method="post">
<p>You are about to delete the tag
<a href="/tags/edit/{{.Data}}">{{.Data}}</a>.
<form class="form-horizontal" action="/tags/del/{{.Data}}" method="post">
<input type="hidden" name="token" value="{{.Token}}">
<div class="form-group">
<label class="col-form-label" for="name">Name</label>
<input class="form-control input-sm" type="text" id="name" name="name" disabled>
</div>
<div class="form-group">
<label class="col-form-label" for="regex">Regex</label>
<input class="form-control input-sm" type="text" id="regex" name="regex" value="{{.Data.Regex}}">
</div>
<button class="btn btn-sm btn-primary" type="submit">Update</button>
<button class="btn btn-sm btn-danger" type="submit">Delete</button>
<a href="/tags" class="btn btn-sm btn-primary">Back</a>
</form>
</div>
@ -459,21 +474,25 @@ const (
{{end}}`
tagsEdit = `{{define "body"}}
<div class="row">
<div class="col-xs-4 offset-4">
<div class="col-8 offset-2">
<div class="card border-primary mx-auto">
<div class="card-header">
<strong>{{.BodyTitle}}</strong>
</div>
<div class="card-body">
<form class="form-horizontal" action="/tags/{{.Data.Name}}/edit" method="post">
<div class="alert alert-dismissible alert-primary">
<strong>Note</strong> Editing a tag does not update tags already assigned to a document.
You need to reindex all documents to avoid false-positive searches.
</div>
<form class="form-horizontal" action="/tags/edit/{{.Data.Name}}" method="post">
<input type="hidden" name="token" value="{{.Token}}">
<div class="form-group">
<label class="col-form-label" for="name">Name</label>
<input class="form-control input-sm" type="text" id="name" name="name" disabled>
<input class="form-control input-sm" type="text" id="name" name="name" value="{{.Data.Name}}" disabled>
</div>
<div class="form-group">
<label class="col-form-label" for="regex">Regex</label>
<input class="form-control input-sm" type="text" id="regex" name="regex" value="{{.Data.Regex}}">
<input class="form-control input-sm mono" type="text" id="regex" name="regex" value="{{.Data.Regex}}" autofocus required>
</div>
<button class="btn btn-sm btn-primary" type="submit">Update</button>
<a href="/tags" class="btn btn-sm btn-primary">Back</a>
@ -484,6 +503,16 @@ const (
</div>
{{end}}`
tagsList = `{{define "body"}}
<div class="page-header">
<div class="row">
<h2>{{.BodyTitle}}</h2>
</div>
</div>
<div class="row mb-3">
<div class="col-xs-5">
<a class="btn btn-sm btn-primary" href="/tags/new"><b>New Tag</b></a>
</div>
</div>
<div class="row">
<table class="table table-stripped table-hover">
<thead>
@ -501,8 +530,8 @@ const (
<td>{{$item.Name}}</td>
<td><code>{{printf "%.50s" $item.Regex}}{{if gt $len 50}}...{{end}}</code></td>
<td><div class="btn-group">
<a href="/tags/{{$item.Name}}/edit" class="btn btn-sm btn-primary">Edit</a>
<a href="/tags/{{$item.Name}}/del" class="btn btn-sm btn-danger">Delete</a>
<a href="/tags/edit/{{$item.Name}}" class="btn btn-sm btn-primary">Edit</a>
<a href="/tags/del/{{$item.Name}}" class="btn btn-sm btn-danger">Delete</a>
</div></td>
</tr>
{{end}}
@ -510,6 +539,32 @@ const (
</tbody>
</table>
</div>
{{end}}`
tagsNew = `{{define "body"}}
<div class="row">
<div class="col-8 offset-2">
<div class="card border-primary mx-auto">
<div class="card-header">
<strong>{{.BodyTitle}}</strong>
</div>
<div class="card-body">
<form class="form-horizontal" action="/tags/new" method="post">
<input type="hidden" name="token" value="{{.Token}}">
<div class="form-group">
<label class="col-form-label" for="name">Name</label>
<input class="form-control input-sm" type="text" id="name" name="name" value="{{.Data.Name}}" autofocus required>
</div>
<div class="form-group">
<label class="col-form-label" for="regex">Regex</label>
<input class="form-control input-sm mono" type="text" id="regex" name="regex" value="{{.Data.Regex}}" required>
</div>
<button class="btn btn-sm btn-primary" type="submit">Create</button>
<a href="/tags" class="btn btn-sm btn-primary">Back</a>
</form>
</div>
</div>
</div>
</div>
{{end}}`
upload = `{{define "body"}}
<div class="row">
@ -723,6 +778,9 @@ code {
background-color: #444444;
border-radius: 2px;
}
embed {
background-color: #999999;
}
textarea,input,select {
color: #e2e2e2 !important;
background-color: #444444 !important;
@ -830,8 +888,11 @@ func (s *HTTPServer) loadTemplates() {
s.templ["userTotpHandler"] = parse(index, menu, userTotp)
// tags
s.templ["tagsHandler"] = parse(index, menu, tagsList)
s.templ["tagsNewHandler"] = parse(index, menu, tagsNew)
s.templ["tagsEditHandler"] = parse(index, menu, tagsEdit)
s.templ["tagsDelHandler"] = parse(index, menu, tagsDel)
// logs
s.templ["logsHandler"] = parse(index, menu, logs)
if fatal {
log.Fatal("parse: ", errors.New("parsing templates failed"))