From c3076c328a277341444938b360922b4d71d7d084 Mon Sep 17 00:00:00 2001 From: ston1th Date: Sat, 24 Aug 2019 14:27:58 +0200 Subject: [PATCH] finished tag implementation --- go.mod | 34 ------- go.sum | 116 ----------------------- pkg/cmd/cmd.go | 2 +- pkg/core/const.go | 1 + pkg/core/helper.go | 5 + pkg/core/types.go | 1 + pkg/db/index.go | 22 ++--- pkg/db/tags.go | 14 ++- pkg/fs/fs.go | 3 +- pkg/fs/helper.go | 1 + pkg/log/scanlog.go | 47 ++++++++++ pkg/scan/helper.go | 25 +++++ pkg/scan/scanner.go | 1 + pkg/server/handler.go | 200 +++++++++++++++++++++++----------------- pkg/server/routes.go | 20 +++- pkg/server/templates.go | 101 ++++++++++++++++---- templates.sh | 3 + templates/custom.css | 3 + templates/dir.html | 7 ++ templates/index.html | 2 +- templates/logs.html | 10 ++ templates/menu.html | 3 + templates/newDir.html | 3 +- templates/search.html | 4 +- templates/tagsDel.html | 14 +-- templates/tagsEdit.html | 12 ++- templates/tagsList.html | 14 ++- templates/tagsNew.html | 26 ++++++ 28 files changed, 397 insertions(+), 297 deletions(-) delete mode 100644 go.mod delete mode 100644 go.sum create mode 100644 pkg/log/scanlog.go create mode 100644 templates/logs.html create mode 100644 templates/tagsNew.html diff --git a/go.mod b/go.mod deleted file mode 100644 index 3eb1a37..0000000 --- a/go.mod +++ /dev/null @@ -1,34 +0,0 @@ -module git.giftfish.de/ston1th/docstore - -go 1.12 - -require ( - git.giftfish.de/ston1th/godrop/v2 v2.1.0 - git.giftfish.de/ston1th/jwt/v3 v3.2.0 - github.com/RoaringBitmap/roaring v0.4.17 // indirect - github.com/Smerity/govarint v0.0.0-20150407073650-7265e41f48f1 // indirect - github.com/blevesearch/bleve v0.7.0 - github.com/blevesearch/blevex v0.0.0-20180227211930-4b158bb555a3 // indirect - github.com/blevesearch/go-porterstemmer v1.0.2 // indirect - github.com/blevesearch/segment v0.0.0-20160915185041-762005e7a34f // indirect - github.com/boltdb/bolt v1.3.1 - github.com/boombuler/barcode v1.0.0 // indirect - github.com/couchbase/vellum v0.0.0-20190328134517-462e86d8716b // indirect - github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d // indirect - github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect - github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 // indirect - github.com/edsrzf/mmap-go v1.0.0 // indirect - github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect - github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect - github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect - github.com/golang/protobuf v1.3.1 // indirect - github.com/gorilla/mux v1.7.1 - github.com/jmhodges/levigo v1.0.0 // indirect - github.com/pquerna/otp v1.1.0 - github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001 // indirect - github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2 // indirect - github.com/syndtr/goleveldb v1.0.0 // indirect - github.com/tecbot/gorocksdb v0.0.0-20181010114359-8752a9433481 // indirect - github.com/urfave/cli v1.20.0 - golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 -) diff --git a/go.sum b/go.sum deleted file mode 100644 index 82346ed..0000000 --- a/go.sum +++ /dev/null @@ -1,116 +0,0 @@ -git.giftfish.de/ston1th/godrop/v2 v2.1.0 h1:vR4aWVeMGJ+tHQH0p44esZrSl5dYJzuCmZqhKWP/vwI= -git.giftfish.de/ston1th/godrop/v2 v2.1.0/go.mod h1:z8KA3HP/vB04wddLXZfYVKURYDxBGau9hlHtV61x9mM= -git.giftfish.de/ston1th/jwt/v3 v3.2.0 h1:V6Lh7OjXn4D3HPd/LoDf7w5N7cyiSXEPAQCIfSVZ05c= -git.giftfish.de/ston1th/jwt/v3 v3.2.0/go.mod h1:jDALrW7OKeIaqYuMxs3GWyPRnQZmk/R+Lu9p/Em3Ziw= -github.com/RoaringBitmap/roaring v0.4.17 h1:oCYFIFEMSQZrLHpywH7919esI1VSrQZ0pJXkZPGIJ78= -github.com/RoaringBitmap/roaring v0.4.17/go.mod h1:D3qVegWTmfCaX4Bl5CrBE9hfrSrrXIr8KVNvRsDi1NI= -github.com/Smerity/govarint v0.0.0-20150407073650-7265e41f48f1 h1:G/NOANWMQev0CftoyxQwtRakdyNNNMB3qxkt/tj1HGs= -github.com/Smerity/govarint v0.0.0-20150407073650-7265e41f48f1/go.mod h1:o80NPAib/LOl8Eysqppjj7kkGkqz++eqzYGlvROpDcQ= -github.com/blevesearch/bleve v0.7.0 h1:znyZ3zjsh2Scr60vszs7rbF29TU6i1q9bfnZf1vh0Ac= -github.com/blevesearch/bleve v0.7.0/go.mod h1:Y2lmIkzV6mcNfAnAdOd+ZxHkHchhBfU/xroGIp61wfw= -github.com/blevesearch/blevex v0.0.0-20180227211930-4b158bb555a3 h1:U6vnxZrTfItfiUiYx0lf/LgHjRSfaKK5QHSom3lEbnA= -github.com/blevesearch/blevex v0.0.0-20180227211930-4b158bb555a3/go.mod h1:WH+MU2F4T0VmSdaPX+Wu5GYoZBrYWdOZWSjzvYcDmqQ= -github.com/blevesearch/go-porterstemmer v1.0.2 h1:qe7n69gBd1OLY5sHKnxQHIbzn0LNJA4hpAf+5XDxV2I= -github.com/blevesearch/go-porterstemmer v1.0.2/go.mod h1:haWQqFT3RdOGz7PJuM3or/pWNJS1pKkoZJWCkWu0DVA= -github.com/blevesearch/segment v0.0.0-20160915185041-762005e7a34f h1:kqbi9lqXLLs+zfWlgo1PIiRQ86n33K1JKotjj4rSYOg= -github.com/blevesearch/segment v0.0.0-20160915185041-762005e7a34f/go.mod h1:IInt5XRvpiGE09KOk9mmCMLjHhydIhNPKPPFLFBB7L8= -github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= -github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= -github.com/boombuler/barcode v1.0.0 h1:s1TvRnXwL2xJRaccrdcBQMZxq6X7DvsMogtmJeHDdrc= -github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= -github.com/couchbase/vellum v0.0.0-20190328134517-462e86d8716b h1:WgO2yt0pCXyPgU3K/pnY1qO/8FP8TxBAaYXqa+BQpx0= -github.com/couchbase/vellum v0.0.0-20190328134517-462e86d8716b/go.mod h1:prYTC8EgTu3gwbqJihkud9zRXISvyulAplQ6exdCo1g= -github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d h1:SwD98825d6bdB+pEuTxWOXiSjBrHdOl/UVp75eI7JT8= -github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= -github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= -github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= -github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 h1:MZRmHqDBd0vxNwenEbKSQqRVT24d3C05ft8kduSwlqM= -github.com/cznic/strutil v0.0.0-20181122101858-275e90344537/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= -github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ= -github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= -github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= -github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= -github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y= -github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 h1:Ujru1hufTHVb++eG6OuNDKMxZnGIvF6o/u8q/8h2+I4= -github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= -github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493 h1:OTanQnFt0bi5iLFSdbEVA/idR6Q2WhCm+deb7ir2CcM= -github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/mux v1.7.1 h1:Dw4jY2nghMMRsh1ol8dv1axHkDwMQK2DHerMNJsIpJU= -github.com/gorilla/mux v1.7.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= -github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY= -github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= -github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pquerna/otp v1.1.0 h1:q2gMsMuMl3JzneUaAX1MRGxLvOG6bzXV51hivBaStf0= -github.com/pquerna/otp v1.1.0/go.mod h1:Zad1CMQfSQZI5KLpahDiSUX4tMMREnXw98IvL1nhgMk= -github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001 h1:YDeskXpkNDhPdWN3REluVa46HQOVuVkjkd2sWnrABNQ= -github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff h1:86HlEv0yBCry9syNuylzqznKXDK11p6D0DT596yNMys= -github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs= -github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2 h1:JNEGSiWg6D3lcBCMCBqN3ELniXujt+0QNHLhNnO0w3s= -github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2/go.mod h1:mjqs7N0Q6m5HpR7QfXVBZXZWSqTjQLeTujjA/xUp2uw= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= -github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/tecbot/gorocksdb v0.0.0-20181010114359-8752a9433481 h1:HOxvxvnntLiPn123Fk+twfUhCQdMDaqmb0cclArW0T0= -github.com/tecbot/gorocksdb v0.0.0-20181010114359-8752a9433481/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= -github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= -github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= -github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc= -github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo= -golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026144532-2772b66316d2/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/cmd/cmd.go b/pkg/cmd/cmd.go index 15802b5..10801de 100644 --- a/pkg/cmd/cmd.go +++ b/pkg/cmd/cmd.go @@ -253,7 +253,7 @@ func serverFlags() []cli.Flag { }, cli.StringSliceFlag{ Name: "langs", - Usage: "languages for tesseract ocr", + Usage: "languages for tesseract ocr (use --langs multiple times to add more languages)", }, cli.IntFlag{ Name: "timeout, t", diff --git a/pkg/core/const.go b/pkg/core/const.go index 8ae9fa2..92043c5 100644 --- a/pkg/core/const.go +++ b/pkg/core/const.go @@ -8,6 +8,7 @@ const ( LoginURI = "/login" LogoutURI = "/logout" SearchURI = "/search" + LogsURI = "/logs" TagsURI = "/tags" UserURI = "/user" diff --git a/pkg/core/helper.go b/pkg/core/helper.go index fb403d2..37c61ae 100644 --- a/pkg/core/helper.go +++ b/pkg/core/helper.go @@ -9,6 +9,7 @@ import ( const ( fileFmt = "20060102_150405" + logFmt = "2006/01/02 15:04:05: " ) var m sync.Mutex @@ -21,3 +22,7 @@ func FileName() string { }() return time.Now().Format(fileFmt) } + +func LogTime() string { + return time.Now().Format(logFmt) +} diff --git a/pkg/core/types.go b/pkg/core/types.go index d05e5b7..d3ee608 100644 --- a/pkg/core/types.go +++ b/pkg/core/types.go @@ -30,6 +30,7 @@ type Path struct { Abs string Flag bool Scan bool + Tags []string } type Paths []Path diff --git a/pkg/db/index.go b/pkg/db/index.go index 7f8eb69..fe32cfb 100644 --- a/pkg/db/index.go +++ b/pkg/db/index.go @@ -11,19 +11,19 @@ const ( pathPrefix = "path/" ) -func (db *DB) GetPaths(res []core.Result) error { +func (db *DB) GetPaths(res []core.Result) { for i, v := range res { err := db.store.Get(pathPrefix+v.ID, &res[i].Path) if err != nil { - return err + res[i].Path = err.Error() } } - return nil } func (db *DB) GetIndexed(p core.Paths) { for i, v := range p { p[i].Flag = db.IsIndexed(v.Abs) + p[i].Tags, _ = db.GetTag(v.Abs) } } @@ -65,22 +65,14 @@ func (db *DB) MoveFile(op, np string) (err error) { if !db.IsIndexed(op) { return nil } - id, err := db.DeleteFile(op) + id, tags, err := db.DeleteFile(op) if err != nil { return } - tags, err := db.GetTag(op) - if err != nil { - return - } - err = db.NewFile(id, np, tags) - if err != nil { - return - } - return db.UnsetTag(op) + return db.NewFile(id, np, tags) } -func (db *DB) DeleteFile(path string) (id string, err error) { +func (db *DB) DeleteFile(path string) (id string, tags []string, err error) { if !db.IsIndexed(path) { return } @@ -92,7 +84,7 @@ func (db *DB) DeleteFile(path string) (id string, err error) { if err != nil { return } - err = db.UnsetTag(path) + tags, err = db.UnsetTag(path) if err != nil { return } diff --git a/pkg/db/tags.go b/pkg/db/tags.go index 162ba48..d887d7c 100644 --- a/pkg/db/tags.go +++ b/pkg/db/tags.go @@ -41,6 +41,11 @@ func (db *DB) GetAllTags() (tags []core.Tag, err error) { return } +func (db *DB) GetTags(name string) (t core.Tag, err error) { + err = db.store.Get(tagsPrefix+name, &t) + return +} + func (db *DB) DeleteTag(name string) error { return db.store.Delete(tagsPrefix + name) } @@ -64,6 +69,11 @@ func (db *DB) SetTag(path string, tags []string) error { return db.store.Set(tagPrefix+path, tags) } -func (db *DB) UnsetTag(path string) error { - return db.store.Delete(tagPrefix + path) +func (db *DB) UnsetTag(path string) (tags []string, err error) { + tags, err = db.GetTag(path) + if err != nil { + return + } + err = db.store.Delete(tagPrefix + path) + return } diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index 4413130..5e9e7fc 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -26,6 +26,7 @@ type Directory struct { type Filesystem struct { Base string + Log *log.ScanLog // protects scan m sync.RWMutex @@ -53,7 +54,7 @@ func NewFilesystem(base string) (fs *Filesystem, err error) { err = errors.New("base dir '" + base + "' is not a directory") return } - fs = &Filesystem{Base: base, scan: make(map[string]struct{})} + fs = &Filesystem{Base: base, Log: log.NewScanLog(0), scan: make(map[string]struct{})} return } diff --git a/pkg/fs/helper.go b/pkg/fs/helper.go index b7b153f..44521e6 100644 --- a/pkg/fs/helper.go +++ b/pkg/fs/helper.go @@ -25,6 +25,7 @@ var reserved = []string{ core.LoginURI, core.LogoutURI, core.SearchURI, + core.LogsURI, core.TagsURI, core.UserURI, core.RawPrefix, diff --git a/pkg/log/scanlog.go b/pkg/log/scanlog.go new file mode 100644 index 0000000..53994f4 --- /dev/null +++ b/pkg/log/scanlog.go @@ -0,0 +1,47 @@ +package log + +import ( + "fmt" + "git.giftfish.de/ston1th/docstore/pkg/core" + "sync" +) + +const defaultSize = 500 + +type ScanLog struct { + // protects logs + sync.RWMutex + + size int + logs []string +} + +func NewScanLog(size int) *ScanLog { + if size <= 0 { + size = defaultSize + } + return &ScanLog{size: size} +} + +func (l *ScanLog) Printf(f string, v ...interface{}) { + s := fmt.Sprintf(f, v...) + Println(s) + s = core.LogTime() + s + l.Lock() + logsLen := len(l.logs) + if logsLen < l.size { + l.logs = append(l.logs, s) + } else if logsLen == l.size { + l.logs = append(l.logs[1:], s) + } + l.Unlock() +} + +func (l *ScanLog) Logs() (logs string) { + l.RLock() + for _, v := range l.logs { + logs += "\n" + v + } + l.RUnlock() + return +} diff --git a/pkg/scan/helper.go b/pkg/scan/helper.go index 15516d0..31faf5f 100644 --- a/pkg/scan/helper.go +++ b/pkg/scan/helper.go @@ -3,10 +3,35 @@ package scan import ( + "net/http" + "os" "path/filepath" "strings" ) +const ( + pdfContentType = "application/pdf" + maxSniffSize = 512 +) + func isPDF(path string) bool { + return checkContentType(path) || checkExtension(path) +} + +func checkContentType(path string) bool { + f, err := os.Open(path) + if err != nil { + return false + } + defer f.Close() + buf := make([]byte, maxSniffSize) + _, err = f.Read(buf) + if err != nil { + return false + } + return http.DetectContentType(buf) == pdfContentType +} + +func checkExtension(path string) bool { return strings.ToLower(filepath.Ext(path)) == pdfExt } diff --git a/pkg/scan/scanner.go b/pkg/scan/scanner.go index 7f14b09..ca01f7d 100644 --- a/pkg/scan/scanner.go +++ b/pkg/scan/scanner.go @@ -76,6 +76,7 @@ func New(c core.Config) (s *Scanner, err error) { langs := c.Langs if langs == nil || llen == 0 { langs = defaultLangs + llen = len(langs) } for i, l := range langs { lang += l diff --git a/pkg/server/handler.go b/pkg/server/handler.go index b43de37..341c0d9 100644 --- a/pkg/server/handler.go +++ b/pkg/server/handler.go @@ -174,49 +174,45 @@ func rawHandler(ctx *Context) { http.ServeContent(ctx.Response, ctx.Request, fi.Name(), fi.ModTime(), f) } +func scanFile(ctx *Context, path string) (err error) { + err = ctx.Srv.FS.AddScan(path) + if err != nil { + return + } + go func() { + defer ctx.Srv.FS.RemoveScan(path) + l := ctx.Srv.FS.Log + file, txt, err := ctx.Srv.Scanner.Scan(path) + if err != nil { + l.Printf("scanner: %s: %s", path, err) + return + } + t, err := ctx.Srv.DB.GetAllTags() + if err != nil { + l.Printf("getTags: %s: %s", path, err) + } + found := tags.Find(txt, t) + id, err := ctx.Srv.DB.Index.Add(txt, found) + if err != nil { + l.Printf("index: %s: %s", path, err) + return + } + err = ctx.Srv.DB.NewFile(id, file, found) + if err != nil { + l.Printf("newFile: %s: %s", path, err) + } + }() + return +} + func indexHandler(ctx *Context) { path := strings.TrimPrefix(ctx.Path(), core.IndexPrefix) switch ctx.Method() { case "GET": - earlyErr := make(chan error) - go func() { - err := ctx.Srv.FS.AddScan(path) - if err != nil { - select { - case earlyErr <- err: - default: - } - return - } - defer ctx.Srv.FS.RemoveScan(path) - file, txt, err := ctx.Srv.Scanner.Scan(path) - if err != nil { - select { - case earlyErr <- err: - default: - } - log.Printf("scanner: %s: %s", path, err) - return - } - t, err := ctx.Srv.DB.GetAllTags() - if err != nil { - log.Printf("getTags: %s: %s", path, err) - } - found := tags.Find(txt, t) - id, err := ctx.Srv.DB.Index.Add(txt, found) - if err != nil { - log.Printf("index: %s: %s", path, err) - return - } - err = ctx.Srv.DB.NewFile(id, file, found) - if err != nil { - log.Printf("newFile: %s: %s", path, err) - } - }() + err := scanFile(ctx, path) p, _ := filepath.Split(path) p = filepath.Dir(p) - select { - case err := <-earlyErr: + if err != nil { ctx.Template("iseHandler") ctx.Data = webData{ Data: template.HTML(strings.ReplaceAll(err.Error(), "\n", "
")), @@ -225,7 +221,6 @@ func indexHandler(ctx *Context) { ctx.Status(http.StatusInternalServerError) ctx.Exec() return - case <-time.After(time.Second): } ctx.Redirect(p, http.StatusFound) } @@ -383,7 +378,7 @@ func deleteHandler(ctx *Context) { clean := ctx.Form("clean") if clean == "true" { f := fs.Clean(path) - id, err := ctx.Srv.DB.DeleteFile(f) + id, _, err := ctx.Srv.DB.DeleteFile(f) if err != nil { log.Printf("db: delete: %s %s", f, err) } @@ -405,7 +400,7 @@ func deleteHandler(ctx *Context) { return } for _, f := range files { - id, err := ctx.Srv.DB.DeleteFile(f) + id, _, err := ctx.Srv.DB.DeleteFile(f) if err != nil { log.Printf("db: delete: %s %s", f, err) } @@ -581,11 +576,7 @@ func searchHandler(ctx *Context) { ctx.Error(err) return } - err = ctx.Srv.DB.GetPaths(res) - if err != nil { - ctx.Error(err) - return - } + ctx.Srv.DB.GetPaths(res) ctx.Data.Data = res ctx.Exec() } @@ -693,71 +684,117 @@ func userTotpHandler(ctx *Context) { } } +func logsHandler(ctx *Context) { + ctx.Template("logsHandler") + ctx.Data = webData{ + Title: "Scan Logs", + BodyTitle: "Scan Logs", + } + switch ctx.Method() { + case "GET": + ctx.Data.Data = ctx.Srv.FS.Log.Logs() + ctx.Exec() + } +} + func tagsHandler(ctx *Context) { ctx.Template("tagsHandler") ctx.Data = webData{ Title: "Tags", BodyTitle: "Tags", } - t, err := ctx.Srv.DB.GetAllTags() - if err != nil { - ctx.Error(err) - return - } - ctx.Data.Data = t switch ctx.Method() { case "GET": + t, err := ctx.Srv.DB.GetAllTags() + if err != nil { + ctx.Error(err) + return + } + ctx.Data.Data = t ctx.Exec() } } +func tagsNewHandler(ctx *Context) { + ctx.Template("tagsNewHandler") + ctx.Data = webData{ + Title: "New Tag", + BodyTitle: "New Tag", + } + switch ctx.Method() { + case "GET": + ctx.Data.Data = core.Tag{} + ctx.Exec() + case "POST": + if !ctx.CheckXsrf() { + return + } + var t core.Tag + t.Name = ctx.Form("name") + t.Regex = ctx.Form("regex") + ctx.Data.Data = t + if t.Name == "" { + ctx.Error("name cannot be empty") + return + } + if t.Regex == "" { + ctx.Error("regex cannot be empty") + return + } + if err := ctx.Srv.DB.NewTag(t.Name, t.Regex); err != nil { + ctx.Error(err) + return + } + log.Printf("tag %s created by: %s\n", t.Name, ctx.User()) + ctx.Redirect(core.TagsURI, http.StatusFound) + } +} + func tagsEditHandler(ctx *Context) { ctx.Template("tagsEditHandler") ctx.Data = webData{ - Title: "Edit User", - BodyTitle: "Edit User", + Title: "Edit Tag", + BodyTitle: "Edit Tag", } - u, err := ctx.Srv.DB.GetUserWithoutPassword() - if err != nil { - ctx.Error(err) - return - } - ctx.Data.Data = u + name := ctx.Var("tag") switch ctx.Method() { case "GET": + t, err := ctx.Srv.DB.GetTags(name) + if err != nil { + ctx.Error(err) + return + } + ctx.Data.Data = t ctx.Exec() case "POST": if !ctx.CheckXsrf() { return } - password := ctx.Form("password") - repeat := ctx.Form("repeat") - - if password != repeat { - ctx.Error("passwords do not match") + var t core.Tag + t.Name = name + t.Regex = ctx.Form("regex") + ctx.Data.Data = t + if t.Regex == "" { + ctx.Error("regex cannot be empty") return } - if err := ctx.Srv.DB.UpdateUserPassword(password); err != nil { + if err := ctx.Srv.DB.NewTag(t.Name, t.Regex); err != nil { ctx.Error(err) return } - log.Printf("update: %s updated\n", ctx.User()) - ctx.Redirect(core.IndexURI, http.StatusFound) + log.Printf("tag %s updated by: %s\n", t.Name, ctx.User()) + ctx.Redirect(core.TagsURI, http.StatusFound) } } func tagsDelHandler(ctx *Context) { ctx.Template("tagsDelHandler") ctx.Data = webData{ - Title: "Edit User", - BodyTitle: "Edit User", + Title: "Delete Tag", + BodyTitle: "Delete Tag", } - u, err := ctx.Srv.DB.GetUserWithoutPassword() - if err != nil { - ctx.Error(err) - return - } - ctx.Data.Data = u + name := ctx.Var("tag") + ctx.Data.Data = name switch ctx.Method() { case "GET": ctx.Exec() @@ -765,19 +802,12 @@ func tagsDelHandler(ctx *Context) { if !ctx.CheckXsrf() { return } - password := ctx.Form("password") - repeat := ctx.Form("repeat") - - if password != repeat { - ctx.Error("passwords do not match") - return - } - if err := ctx.Srv.DB.UpdateUserPassword(password); err != nil { + if err := ctx.Srv.DB.DeleteTag(name); err != nil { ctx.Error(err) return } - log.Printf("update: %s updated\n", ctx.User()) - ctx.Redirect(core.IndexURI, http.StatusFound) + log.Printf("tag %s deleted by: %s\n", name, ctx.User()) + ctx.Redirect(core.TagsURI, http.StatusFound) } } diff --git a/pkg/server/routes.go b/pkg/server/routes.go index 562afb9..e4e5a00 100644 --- a/pkg/server/routes.go +++ b/pkg/server/routes.go @@ -123,21 +123,35 @@ var routes = append(static, []route{ []string{"GET", "POST"}, }, { - "/tags", + core.LogsURI, + jwtHandler( + authHandler( + logsHandler)), + []string{"GET"}, + }, + { + core.TagsURI, jwtHandler( authHandler( tagsHandler)), []string{"GET"}, }, { - "/tags/{tag:[a-zA-Z0-9]+}/edit", + "/tags/new", + jwtHandler( + authHandler( + tagsNewHandler)), + []string{"GET", "POST"}, + }, + { + "/tags/edit/{tag:[a-zA-Z0-9]+}", jwtHandler( authHandler( tagsEditHandler)), []string{"GET", "POST"}, }, { - "/tags/{tag:[a-zA-Z0-9]+}/del", + "/tags/del/{tag:[a-zA-Z0-9]+}", jwtHandler( authHandler( tagsDelHandler)), diff --git a/pkg/server/templates.go b/pkg/server/templates.go index 1166c34..0a072cb 100644 --- a/pkg/server/templates.go +++ b/pkg/server/templates.go @@ -60,6 +60,7 @@ const ( Path Type + Tags Options @@ -69,6 +70,7 @@ const ( {{$item.Name}} Directory +
Move Delete @@ -79,6 +81,11 @@ const ( {{$item.Name}} File + + {{range $t := $item.Tags}} + {{$t}} + {{end}} +
{{if $item.Scan}} Scanning @@ -160,7 +167,7 @@ const ( DocStore | {{.Title}} - + @@ -262,19 +269,32 @@ const (
+{{end}}` + logs = `{{define "body"}} +
+
+

{{.BodyTitle}}

+
+
+
+ +
{{end}}` menu = `{{define "menu"}} +{{if .Login}}
+{{end}} +{{if .Login}}
+{{end}}