diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index acc777e..a881164 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -1,12 +1,13 @@ package cache import ( + "git.giftfish.de/ston1th/gowiki/pkg/core" "sync" "time" ) type CachedPage struct { - Page *Page + Page *core.Page CacheTime time.Time } @@ -19,12 +20,12 @@ func NewCache() *Cache { return &Cache{Cache: make(map[string]*CachedPage)} } -func (c *Cache) Add(title string, p *Page) { +func (c *Cache) Add(title string, p *core.Page) { now := time.Now() c.Lock() defer c.Unlock() cp, ok := c.Cache[title] - if ok && now.After(c.CacheTime) { + if ok && now.After(cp.CacheTime) { cp.Page = p cp.CacheTime = now c.Cache[title] = cp @@ -33,7 +34,7 @@ func (c *Cache) Add(title string, p *Page) { c.Cache[title] = &CachedPage{p, now} } -func (c *Cache) Get(title string) *Page { +func (c *Cache) Get(title string) *core.Page { c.RLock() defer c.RUnlock() if cp, ok := c.Cache[title]; ok { diff --git a/pkg/cmd/cmd.go b/pkg/cmd/cmd.go index 41d3320..97db392 100644 --- a/pkg/cmd/cmd.go +++ b/pkg/cmd/cmd.go @@ -1,19 +1,21 @@ // Copyright (C) 2017 Marius Schellenberger -package main +package cmd import ( "errors" "git.giftfish.de/ston1th/godrop" "github.com/urfave/cli" - logger "log" + stdlog "log" "net" "os" "os/signal" "syscall" "time" + "git.giftfish.de/ston1th/gowiki/pkg/db" "git.giftfish.de/ston1th/gowiki/pkg/log" + "git.giftfish.de/ston1th/gowiki/pkg/server" ) const ( @@ -50,7 +52,7 @@ func server(conf serverConfig) (err error) { return } - srv := NewHTTPServer(l, conf.Version, conf.SecureCookie) + srv := server.NewHTTPServer(l, conf.Version, conf.SecureCookie) err = srv.Start() if err != nil { return @@ -91,7 +93,7 @@ func Run(version string) { Action: func(c *cli.Context) error { conf.Version = app.Version if err := server(conf); err != nil { - logger.Fatal("server: ", err) + stdlog.Fatal("server: ", err) } return nil }, @@ -102,10 +104,10 @@ func Run(version string) { Flags: defaultFlags(dumpRestoreFlags()), Action: func(c *cli.Context) error { if err := os.Chdir(conf.DataDir); err != nil { - logger.Fatal("dump: ", err) + stdlog.Fatal("dump: ", err) } - if err := dump(blevePath, verbose); err != nil { - logger.Fatal("dump: ", err) + if err := db.Dump(blevePath, verbose); err != nil { + stdlog.Fatal("dump: ", err) } return nil }, @@ -116,10 +118,10 @@ func Run(version string) { Flags: defaultFlags(dumpRestoreFlags()), Action: func(c *cli.Context) error { if err := os.Chdir(conf.DataDir); err != nil { - logger.Fatal("restore: ", err) + stdlog.Fatal("restore: ", err) } - if err := restore(blevePath, verbose); err != nil { - logger.Fatal("restore: ", err) + if err := db.Restore(blevePath, verbose); err != nil { + stdlog.Fatal("restore: ", err) } return nil }, diff --git a/pkg/permission/permission.go b/pkg/core/permission.go similarity index 73% rename from pkg/permission/permission.go rename to pkg/core/permission.go index 854ebb3..8c9953a 100644 --- a/pkg/permission/permission.go +++ b/pkg/core/permission.go @@ -1,4 +1,4 @@ -package permission +package core type Permission int @@ -9,19 +9,19 @@ const ( Private ) -func Parse(p int) Permission { +func ParsePerm(p int) Permission { switch p { - case Public: + case 1: return Public - case Internal: + case 2: return Internal - case Private: + case 3: return Private } return Invalid } -func Read(username string, p *Page) bool { +func ReadPerm(username string, p *Page) bool { switch p.Perm { case Public: return true @@ -33,7 +33,7 @@ func Read(username string, p *Page) bool { return false } -func Write(username string, p *Page) bool { +func WritePerm(username string, p *Page) bool { switch p.Perm { case Public, Internal: if p.Owner == "" { diff --git a/pkg/core/types.go b/pkg/core/types.go new file mode 100644 index 0000000..c59b8a4 --- /dev/null +++ b/pkg/core/types.go @@ -0,0 +1,20 @@ +package core + +import "html/template" + +type Page struct { + Title string `json:"title"` + Markdown string `json:"markdown"` + Index template.HTML `json:"index"` + Text template.HTML `json:"text"` + Created string `json:"created"` + Updated string `json:"updated"` + Owner string `json:"owner"` + Perm Permission `json:"perm"` +} + +type Result struct { + Title string + StoreTitle string + Text string +} diff --git a/pkg/db/db.go b/pkg/db/db.go index 5b29d43..b4a15d4 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -3,28 +3,31 @@ package db import ( - "bytes" - "errors" - "git.giftfish.de/ston1th/gowiki/cache" + "fmt" + "git.giftfish.de/ston1th/gowiki/pkg/cache" + "git.giftfish.de/ston1th/gowiki/pkg/index" "github.com/boltdb/bolt" ) const ( - dbFile = "gowiki.db" - defUser = "admin" - defPassword = "gowiki" - userTable = "user" - pageTable = "page" - bcryptCost = 13 - indexName = "Index" - welcome = `# Welcome to GoWiki + dbFile = "gowiki.db" + defUser = "admin" + defPassword = "gowiki" + userTable = "user" + pageTable = "page" + snippetTable = "snippet" + bcryptCost = 13 + indexName = "Index" + welcome = `# Welcome to GoWiki This is the [Index](/wiki/Index) page. You can customize it how you like.` + wikiSection = "wiki" ) -const ( + +/*const ( maxResult = 1e6 maxSearchResult = 101 -) +)*/ func dbErr(i interface{}) error { return fmt.Errorf("db: %s", i) @@ -34,9 +37,13 @@ type BoltStore struct { Marshaler db *bolt.DB cache *cache.Cache + index *index.Index } -func NewBoltStore() (bs *BoltStore, err error) { +// TODO create initial wiki page +// TODO init index + +func NewBoltStore(indexPath string) (bs *BoltStore, err error) { db, err := bolt.Open(dbFile, 0666, nil) if err != nil { return @@ -52,8 +59,12 @@ func NewBoltStore() (bs *BoltStore, err error) { }); err != nil { return } - bs = &BoltStore{NewGOB(), db, cache.NewCache()} - if !bs.UserExists(defUser) { + i, err := index.NewIndex(indexPath) + if err != nil { + return + } + bs = &BoltStore{NewGOB(), db, cache.NewCache(), i} + if bs.UserExists(defUser) == nil { err = bs.CreateUser(defUser, defPassword, true) if err != nil { return diff --git a/pkg/db/dump.go b/pkg/db/dump.go index 213681e..95d9bf9 100644 --- a/pkg/db/dump.go +++ b/pkg/db/dump.go @@ -6,15 +6,17 @@ import ( "encoding/json" "fmt" "os" + + "git.giftfish.de/ston1th/gowiki/pkg/core" ) -type Dump struct { - Articles []DumpArticle `json:"articles"` - Snippets []Snippet `json:"snippets"` - Users []User `json:"users"` +type DBDump struct { + Pages []DumpPage `json:"pages"` + Snippets []Snippet `json:"snippets"` + Users []User `json:"users"` } -type DumpArticle struct { +type DumpPage struct { Title string `json:"title"` LinkTitle string `json:"link_title"` MD string `json:"md"` @@ -29,38 +31,26 @@ func verboseLog(prefix, title string, v bool) { } } -func dump(path string, v bool) (err error) { - i, err := NewDumpIndex(path) - if err != nil { - return - } - defer i.Close() +func Dump(path string, v bool) (err error) { bs, err := NewBoltStore() if err != nil { return } - art, err := i.Get(false, indexName) - if err != nil { - return - } - var dump Dump - titles, err := i.Dump() - if err != nil { - return - } - for _, title := range titles { - art, err = i.Get(false, title) + var dump DBDump + //TODO bs.GetAllPages + var pages []*core.Page + for _, p := range pages { if err != nil { return } - verboseLog("dump:", art.Title, v) - dump.Articles = append(dump.Articles, DumpArticle{ - Title: art.Title, - LinkTitle: art.LinkTitle, - MD: art.MD, - Created: art.Created, - Updated: art.Updated, - Public: art.Public, + verboseLog("dump:", p.Title, v) + dump.Pages = append(dump.Pages, DumpPage{ + Title: p.Title, + LinkTitle: p.LinkTitle, + MD: p.MD, + Created: p.Created, + Updated: p.Updated, + Public: p.Public, }) } //TODO snippets @@ -73,7 +63,7 @@ func dump(path string, v bool) (err error) { return json.NewEncoder(os.Stdout).Encode(dump) } -func restore(path string, v bool) (err error) { +func Restore(path string, v bool) (err error) { i, err := NewIndex(path) if err != nil { return @@ -83,7 +73,7 @@ func restore(path string, v bool) (err error) { if err != nil { return } - var dump Dump + var dump DBDump err = json.NewDecoder(os.Stdin).Decode(&dump) if err != nil { return diff --git a/pkg/db/helper.go b/pkg/db/helper.go index 8e0f360..ae0cd11 100644 --- a/pkg/db/helper.go +++ b/pkg/db/helper.go @@ -1,6 +1,9 @@ package db -import "golang.org/x/crypto/sha3" +import ( + "golang.org/x/crypto/sha3" + "time" +) func validatePassword(pw string) (b []byte) { b = []byte(pw) diff --git a/pkg/db/marshal.go b/pkg/db/marshal.go index 08cbf28..13401d9 100644 --- a/pkg/db/marshal.go +++ b/pkg/db/marshal.go @@ -1,6 +1,9 @@ package db -import "encoding/gob" +import ( + "bytes" + "encoding/gob" +) type Marshaler interface { Marshal(v interface{}) ([]byte, error) diff --git a/pkg/db/page.go b/pkg/db/page.go index 6967d08..a285d72 100644 --- a/pkg/db/page.go +++ b/pkg/db/page.go @@ -1,44 +1,22 @@ package db import ( - "git.giftfish.de/ston1th/gowiki/pkg/cache" - "git.giftfish.de/ston1th/gowiki/pkg/permission" - "git.giftfish.de/ston1th/gowiki/pkg/reneder" + "git.giftfish.de/ston1th/gowiki/pkg/core" + "git.giftfish.de/ston1th/gowiki/pkg/render" + "git.giftfish.de/ston1th/gowiki/pkg/util" + "github.com/boltdb/bolt" ) -type Article struct { - Title string `json:"title"` - LinkTitle string `json:"-"` - Search string `json:"search"` - Index template.HTML `json:"index"` - Text template.HTML `json:"text"` - MD string `json:"md"` - Created string `json:"created"` - Updated string `json:"updated"` - Public bool `json:"public"` -} - -type Page struct { - Title string `json:"title"` - Markdown string `json:"markdown"` - Index template.HTML `json:"index"` - Text template.HTML `json:"text"` - Created string `json:"created"` - Updated string `json:"updated"` - Owner string `json:"owner"` - Perm permission.Permission `json:"perm"` -} - var ( errPageNotFound = dbErr("page not found") errInvalidPermission = dbErr("invalid permission") errPrivateWikiPage = dbErr("private wiki pages are not allowed") ) -func (bs *BoltStore) GetPage(title, username string) (p *Page, err error) { +func (bs *BoltStore) GetPage(title, username string) (p *core.Page, err error) { p = bs.cache.Get(title) if p == nil { - p = new(Page) + p = new(core.Page) err = bs.db.View(func(tx *bolt.Tx) error { v := tx.Bucket([]byte(pageTable)).Get([]byte(title)) if v == nil { @@ -47,43 +25,38 @@ func (bs *BoltStore) GetPage(title, username string) (p *Page, err error) { return bs.Unmarshal(v, p) }) if err != nil { - return err + return } bs.cache.Add(title, p) } - if !permission.Read(username, p) { + if !core.ReadPerm(username, p) { err = errPageNotFound } return } -func (bs *BoltStore) pageExists(title, username string) (page *Page, err error) { - page, err = bs.GetPage(title) - if err != nil { - return - } - if !permission.Verify(username, page) { - err = errPageNotFound - } +// TODO access db not cache +func (bs *BoltStore) pageExists(title, username string) (p *core.Page, err error) { + p, err = bs.GetPage(title, username) return } -func (bs *BoltStore) CreatePage(title, section, markdown, username string, perm int) (page *Page, err error) { - p := permission.Parse(perm) - if p == permission.Invalid { +func (bs *BoltStore) CreatePage(title, section, markdown, username string, perm int) (page *core.Page, err error) { + p := core.ParsePerm(perm) + if p == core.Invalid { err = errInvalidPermission return } - if section == wikiSection && p == permission.Private { + if section == wikiSection && p == core.Private { err = errPrivateWikiPage return } - st := storeTitle(section, title) + st := util.StoreTitle(section, title) if _, err = bs.pageExists(st, username); err != nil { return } - page = &Page{ + page = &core.Page{ Title: title, Markdown: markdown, Created: created(username), @@ -103,25 +76,25 @@ func (bs *BoltStore) CreatePage(title, section, markdown, username string, perm return } - err = index.Add(title, search) + err = bs.index.Add(title, st, search) if err != nil { return } - cache.Add(st, page) + bs.cache.Add(st, page) return } -func (bs *BoltStore) UpdatePage(title, section, markdown, username string, perm int) (page *Page, err error) { - p := permission.Parse(perm) - if p == permission.Invalid { +func (bs *BoltStore) UpdatePage(title, section, markdown, username string, perm int) (page *core.Page, err error) { + p := core.ParsePerm(perm) + if p == core.Invalid { err = errInvalidPermission return } - if section == wikiSection && p == permission.Private { + if section == wikiSection && p == core.Private { err = errPrivateWikiPage return } - st := storeTitle(section, title) + st := util.StoreTitle(section, title) page, err = bs.pageExists(st, username) if err != nil { return @@ -143,20 +116,23 @@ func (bs *BoltStore) UpdatePage(title, section, markdown, username string, perm return } - err = index.Add(title, search) + err = bs.index.Add(title, st, search) if err != nil { return } - cache.Add(st, page) + bs.cache.Add(st, page) return } func (bs *BoltStore) DeletePage(title, section, username string) error { - st := storeTitle(section, title) + st := util.StoreTitle(section, title) if _, err := bs.pageExists(st, username); err != nil { return err } - return bs.db.Update(func(tx *bolt.Tx) error { + if err := bs.db.Update(func(tx *bolt.Tx) error { return tx.Bucket([]byte(pageTable)).Delete([]byte(st)) - }) + }); err != nil { + return err + } + return bs.index.Delete(st) } diff --git a/pkg/db/snippet.go b/pkg/db/snippet.go index 6ed1549..98a3533 100644 --- a/pkg/db/snippet.go +++ b/pkg/db/snippet.go @@ -2,6 +2,7 @@ package db import ( "bytes" + "git.giftfish.de/ston1th/gowiki/pkg/render" "github.com/boltdb/bolt" "html/template" "strings" @@ -25,9 +26,8 @@ func copyBuf(buf []byte) (b []byte) { } var ( - errSnippetNotExists = dbErr("snippet does not exist") - errSnippetExists = dbErr("snippet already exist") - errInvalidPermission = dbErr("invalid permission") + errSnippetNotExists = dbErr("snippet does not exist") + errSnippetExists = dbErr("snippet already exist") ) func (bs *BoltStore) GetSnippets(username string) (snip []Snippet, err error) { @@ -100,7 +100,7 @@ func checkPerm(p int) error { } func (bs *BoltStore) CreateSnippet(username, title, text string, perm int) (newTitle string, err error) { - newTitle, title = makeLinkTitle(title) + //newTitle, title = makeLinkTitle(title) err = checkPerm(perm) if err != nil { return @@ -111,7 +111,7 @@ func (bs *BoltStore) CreateSnippet(username, title, text string, perm int) (newT return } err = bs.db.Update(func(tx *bolt.Tx) error { - snip := render(text).Snippet() + snip := render.Render(text).Snippet() snip.Title = title snip.Created = username + " " + now() snip.Permission = perm @@ -134,7 +134,7 @@ func (bs *BoltStore) EditSnippet(username, title, text string, perm int) (err er return } err = bs.db.Update(func(tx *bolt.Tx) error { - snip := render(text).Snippet() + snip := render.Render(text).Snippet() snip.Title = title snip.Created = username + " " + now() snip.Permission = perm diff --git a/pkg/db/user.go b/pkg/db/user.go index 6f3d968..a4c3b35 100644 --- a/pkg/db/user.go +++ b/pkg/db/user.go @@ -5,6 +5,13 @@ import ( "golang.org/x/crypto/bcrypt" ) +type User struct { + Username string + Password string + Admin bool + Locked int +} + var ( errUserNotFound = dbErr("user not found") errUserExists = dbErr("user already exist") @@ -28,7 +35,7 @@ func (bs *BoltStore) GetUser(username string) (u User, err error) { err = bs.db.View(func(tx *bolt.Tx) error { v := tx.Bucket([]byte(userTable)).Get([]byte(username)) if v == nil { - return errUserNotExists + return errUserNotFound } return bs.Unmarshal(v, &u) }) @@ -47,7 +54,7 @@ func (bs *BoltStore) UserIsAdmin(username string) (admin bool) { func (bs *BoltStore) UserExists(username string) error { return bs.db.View(func(tx *bolt.Tx) error { if tx.Bucket([]byte(userTable)).Get([]byte(username)) == nil { - return errUserNotExists + return errUserNoFound } return nil }) diff --git a/pkg/index/index.go b/pkg/index/index.go index 455dd21..5a24a38 100644 --- a/pkg/index/index.go +++ b/pkg/index/index.go @@ -1,4 +1,4 @@ -// Copyright (C) 2017 Marius Schellenberger +// Copyright (C) 2018 Marius Schellenberger package index @@ -6,13 +6,19 @@ import ( "fmt" "os" + "git.giftfish.de/ston1th/gowiki/pkg/core" "github.com/blevesearch/bleve" - "github.com/blevesearch/bleve/document" + // "github.com/blevesearch/bleve/document" +) + +const ( + maxSearchResult = 100 ) type indexPage struct { - Title string `json:"title"` - Search string `json:"search"` + Title string `json:"title"` + StoreTitle string `json:"store_title"` + Search string `json:"search"` } func indexErr(i interface{}) error { @@ -32,21 +38,14 @@ func NewIndex(path string) (i *Index, err error) { err = indexErr(err) return } - i = &Index{index: bi} - _, err = i.Index(indexName, welcome, defUser, true) - if err != nil { - err = indexErr(err) - return - } } else { bi, err = bleve.Open(path) if err != nil { err = indexErr(err) return } - i = &Index{index: bi} } - i.cache = NewIndexCache() + i = &Index{i: bi} return } @@ -54,7 +53,7 @@ func (i *Index) Close() error { return i.i.Close() } -func (i *Index) Add(title, search string) error { +func (i *Index) Add(title, storeTitle, search string) error { doc, _ := i.i.Document(title) if doc != nil { err := i.Delete(title) @@ -62,19 +61,14 @@ func (i *Index) Add(title, search string) error { return err } } - return i.i.Index(title, indexPage{title, search}) + return i.i.Index(storeTitle, indexPage{title, storeTitle, search}) } -func (i *Index) Delete(title string) error { - return i.i.Delete(title) +func (i *Index) Delete(storeTitle string) error { + return i.i.Delete(storeTitle) } -type Result struct { - Title string - Text string -} - -func (i *Index) Search(search string) (results []Result, err error) { +func (i *Index) Search(search string) (results []core.Result, err error) { req := bleve.NewSearchRequest(bleve.NewQueryStringQuery("title:" + search + " search:" + search)) req.Highlight = bleve.NewHighlightWithStyle("html") req.Size = maxSearchResult @@ -86,17 +80,22 @@ func (i *Index) Search(search string) (results []Result, err error) { if hit == nil { continue } - title := hit.ID - text := "" + r := core.Result{Title: hit.ID} for fragField, frags := range hit.Fragments { + if fragField == "store_title" { + for _, f := range frags { + r.StoreTitle += f + } + break + } if fragField == "title" || fragField == "search" { for _, f := range frags { - text += f + r.Text += f } break } } - results = append(results, Result{title, text}) + results = append(results, r) } return } diff --git a/pkg/log/log.go b/pkg/log/log.go index 660cd5f..0fb6fa7 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -1,25 +1,25 @@ // Copyright (C) 2017 Marius Schellenberger -package main +package log import ( - logger "log" + stdlog "log" "os" ) -var log *logger.Logger +var log *stdlog.Logger -func newLogger(file string) { +func NewLogger(file string) { if file == "" { - log = logger.New(os.Stdout, "", logger.LstdFlags) + log = stdlog.New(os.Stdout, "", stdlog.LstdFlags) return } f, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644) if err != nil { - logger.Fatal(err) + stdlog.Fatal(err) } - logger.SetOutput(f) - log = logger.New(f, "", logger.LstdFlags) + stdlog.SetOutput(f) + log = stdlog.New(f, "", stdlog.LstdFlags) } func Println(v ...interface{}) { diff --git a/pkg/render/render.go b/pkg/render/render.go index 8fda177..b4a822b 100644 --- a/pkg/render/render.go +++ b/pkg/render/render.go @@ -8,6 +8,7 @@ import ( "strconv" "strings" + "git.giftfish.de/ston1th/gowiki/pkg/core" "github.com/blevesearch/bleve/analysis" htmlFilter "github.com/blevesearch/bleve/analysis/char_filters/html_char_filter" "github.com/russross/blackfriday" @@ -28,7 +29,7 @@ var ( titleRe = regexp.MustCompile("[^a-zA-Z0-9 -]+") ) -func Render(p *Page) string { +func Render(p *core.Page) string { rend := blackfriday.MarkdownCommon([]byte(p.Markdown)) index, html := buildIndex(string(rend)) html = strings.Replace(html, "