ston1th/gowiki
0
0
Fork 0
Code Issues 2 Releases Activity

added dump/restore and fixed title stuff

Browse source
This commit is contained in:
ston1th 2018-10-16 21:26:39 +02:00
commit 8d3187b160
13 changed files with 200627 additions and 189988 deletions
Download patch file Download diff file Expand all files Collapse all files

82
pkg/cmd/cmd.go
View file

@ -5,6 +5,7 @@ package cmd
import (
"git.giftfish.de/ston1th/godrop/v2"
"git.giftfish.de/ston1th/gowiki/pkg/core"
"git.giftfish.de/ston1th/gowiki/pkg/db"
"git.giftfish.de/ston1th/gowiki/pkg/log"
"git.giftfish.de/ston1th/gowiki/pkg/server"
"github.com/urfave/cli"
@ -18,13 +19,18 @@ import (
const (
defDataDir = "/var/lib/gowiki"
defListen = "127.0.0.1:8090"
defListen = "127.0.0.1:8080"
defLogFile = "gowiki.log"
defRunUser = "wiki"
defRunGroup = "wiki"
)
var (
conf core.Config
dumpFile string
)
func initServer(conf core.Config) (err error) {
dropCfg := godrop.Config{
User: conf.User,
@ -70,11 +76,6 @@ func initServer(conf core.Config) (err error) {
return
}
var (
verbose bool
conf core.Config
)
func Run(version string) {
app := cli.NewApp()
app.Name = "gowiki"
@ -95,31 +96,67 @@ func Run(version string) {
},
{
Name: "dump",
Usage: "create a json dump of all articles in the data directory (dump is written to stdout) gowiki needs to be stopped",
Usage: "create a json dump of all articles in the data directory (gowiki needs to be stopped)",
Flags: defaultFlags(dumpRestoreFlags()),
Action: func(c *cli.Context) error {
if err := os.Chdir(conf.DataDir); err != nil {
var (
file = os.Stdout
err error
)
if dumpFile != "-" {
file, err = os.OpenFile(dumpFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0640)
if err != nil {
stdlog.Fatal("dump: ", err)
}
}
if err = os.Chdir(conf.DataDir); err != nil {
stdlog.Fatal("dump: ", err)
}
DB, err := db.New(conf.DataDir)
if err != nil {
stdlog.Fatal("dump: ", err)
}
if err = DB.Dump(file); err != nil {
stdlog.Fatal("dump: ", err)
}
file.Close()
err = DB.Close()
if err != nil {
stdlog.Fatal("dump: ", err)
}
//TODO
//if err := db.Dump(blevePath, verbose); err != nil {
// stdlog.Fatal("dump: ", err)
//}
return nil
},
},
{
Name: "restore",
Usage: "restore a json dump to the data directory (dump is read from stdin) gowiki needs to be stopped",
Usage: "restore a json dump to the data directory (gowiki needs to be stopped)",
Flags: defaultFlags(dumpRestoreFlags()),
Action: func(c *cli.Context) error {
if err := os.Chdir(conf.DataDir); err != nil {
var (
file = os.Stdin
err error
)
if dumpFile != "-" {
file, err = os.Open(dumpFile)
if err != nil {
stdlog.Fatal("dump: ", err)
}
}
if err = os.Chdir(conf.DataDir); err != nil {
stdlog.Fatal("restore: ", err)
}
DB, err := db.New(conf.DataDir)
if err != nil {
stdlog.Fatal("restore: ", err)
}
if err = DB.Restore(file); err != nil {
stdlog.Fatal("restore: ", err)
}
file.Close()
err = DB.Close()
if err != nil {
stdlog.Fatal("restore: ", err)
}
//TODO
//if err := db.Restore(blevePath, verbose); err != nil {
// stdlog.Fatal("restore: ", err)
//}
return nil
},
},
@ -178,10 +215,11 @@ func serverFlags() []cli.Flag {
func dumpRestoreFlags() []cli.Flag {
return []cli.Flag{
cli.BoolFlag{
Name: "verbose, v",
Usage: "enable verbose mode",
Destination: &verbose,
cli.StringFlag{
Name: "file, f",
Value: "-",
Usage: "file to read/write dump (default stdin/stdout)",
Destination: &dumpFile,
},
}
}

28
pkg/db/db.go
View file

@ -7,7 +7,9 @@ import (
"git.giftfish.de/ston1th/gowiki/pkg/cache"
"git.giftfish.de/ston1th/gowiki/pkg/core"
"git.giftfish.de/ston1th/gowiki/pkg/index"
"git.giftfish.de/ston1th/gowiki/pkg/render"
"git.giftfish.de/ston1th/gowiki/pkg/store"
"io"
)
const (
@ -61,10 +63,30 @@ func New(datadir string) (db *DB, err error) {
return
}
func (db *DB) Close() error {
err := db.store.Close()
func (db *DB) Dump(w io.Writer) error {
return db.store.Dump(w)
}
func (db *DB) Restore(r io.Reader) (err error) {
err = db.store.Restore(r)
if err != nil {
return err
return
}
err = db.store.ForEach(func(k string, _ []byte) error {
if trim, ok := hasTrimPrefix(k, pagePrefix); ok {
p := new(core.Page)
db.store.Get(pagePrefix+trim, p)
db.Index.Add(p.Title, p.StoreTitle, render.Render(p))
}
return nil
})
return
}
func (db *DB) Close() (err error) {
err = db.store.Close()
if err != nil {
return
}
return db.Index.Close()
}

93
pkg/db/dump.go
View file

@ -1,93 +0,0 @@
// Copyright (C) 2018 Marius Schellenberger
package db
/*import (
"encoding/json"
"fmt"
"os"
"git.giftfish.de/ston1th/gowiki/pkg/core"
)
type DBDump struct {
Pages []DumpPage `json:"pages"`
Snippets []Snippet `json:"snippets"`
Users []User `json:"users"`
}
type DumpPage struct {
Title string `json:"title"`
LinkTitle string `json:"link_title"`
MD string `json:"md"`
Created string `json:"created"`
Updated string `json:"updated"`
Public bool `json:"public"`
}
func verboseLog(prefix, title string, v bool) {
if v {
fmt.Fprintln(os.Stderr, prefix, title)
}
}
func Dump(path string, v bool) (err error) {
bs, err := NewBoltStore()
if err != nil {
return
}
var dump DBDump
//TODO bs.GetAllPages
var pages []*core.Page
for _, p := range pages {
if err != nil {
return
}
verboseLog("dump:", p.Title, v)
dump.Pages = append(dump.Pages, DumpPage{
Title: p.Title,
LinkTitle: p.LinkTitle,
MD: p.MD,
Created: p.Created,
Updated: p.Updated,
Public: p.Public,
})
}
//TODO snippets
//verboseLog("dump:", "snippets", v)
verboseLog("dump:", "users", v)
dump.Users, err = bs.DumpUsers()
if err != nil {
return
}
return json.NewEncoder(os.Stdout).Encode(dump)
}
func Restore(path string, v bool) (err error) {
i, err := NewIndex(path)
if err != nil {
return
}
defer i.Close()
bs, err := NewBoltStore()
if err != nil {
return
}
var dump DBDump
err = json.NewDecoder(os.Stdin).Decode(&dump)
if err != nil {
return
}
for _, art := range dump.Articles {
verboseLog("restore:", art.Title, v)
err = i.Restore(&art)
if err != nil {
return
}
}
//TODO snippets
//verboseLog("restore:", "snippets", v)
verboseLog("restore:", "users", v)
err = bs.RestoreUsers(dump.Users)
return
}*/

15
pkg/db/page.go
View file

@ -25,7 +25,7 @@ func (db *DB) GetAllPages(username string) (pages core.Pages) {
if trim == core.IndexURI {
return nil
}
p, err := db.GetPage(trim, username)
p, err := db.getPage(trim, username)
if err == nil {
pages = append(pages, *p)
}
@ -36,11 +36,12 @@ func (db *DB) GetAllPages(username string) (pages core.Pages) {
return
}
func (db *DB) GetPage(title, username string) (p *core.Page, err error) {
p = db.cache.Get(title)
func (db *DB) GetPage(section, title, username string) (p *core.Page, err error) {
st := render.StoreTitle(section, title)
p = db.cache.Get(st)
if p == nil {
p, err = db.getPage(title, username)
db.cache.Add(title, p)
p, err = db.getPage(st, username)
db.cache.Add(st, p)
return
}
if !core.ReadPerm(username, p) {
@ -74,7 +75,7 @@ func (db *DB) CreatePage(title, section, markdown, username string, p core.Permi
return
}
}
st := render.StoreTitle(section, title)
st := render.StoreTitle(section, render.Title(title))
if _, err = db.getPage(st, username); err != nil && err != store.ErrKeyNotFound {
return
}
@ -130,7 +131,7 @@ func (db *DB) UpdatePage(title, section, markdown, username string, p core.Permi
return
}
err = db.Index.Add(title, st, search)
err = db.Index.Add(page.Title, st, search)
if err != nil {
return
}

41
pkg/db/user.go
View file

@ -8,6 +8,7 @@ import (
"git.giftfish.de/ston1th/gowiki/pkg/log"
"git.giftfish.de/ston1th/jwt/v3"
"golang.org/x/crypto/bcrypt"
"regexp"
"sort"
"strconv"
"strings"
@ -19,26 +20,21 @@ var (
errUserNotFound = dbErr("user not found")
errUserExists = dbErr("user already exist")
errUserReserved = dbErr("username is reserved")
userRe = regexp.MustCompile("[^a-zA-Z0-9]+")
)
func (db *DB) DumpUsers() (users core.Users, err error) {
db.store.ForEach(func(k string, _ []byte) error {
func (db *DB) GetUsers() (users core.Users, err error) {
err = db.store.ForEach(func(k string, _ []byte) error {
if trim, ok := hasTrimPrefix(k, userPrefix); ok {
u, err := db.GetUser(trim)
u.Password = ""
if err == nil {
users = append(users, u)
}
}
return nil
})
return
}
func (db *DB) GetUsers() (users core.Users, err error) {
users, err = db.DumpUsers()
for i := range users {
users[i].Password = ""
}
sort.Sort(users)
return
}
@ -58,31 +54,8 @@ func (db *DB) UserExists(username string) error {
return db.store.Get(userPrefix+username, nil)
}
// TODO
/*func (bs *BoltStore) RestoreUsers(users []User) error {
return bs.db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(userTable))
err := b.ForEach(func(k, v []byte) error {
return b.Delete(k)
})
if err != nil {
return err
}
for _, u := range users {
v, err := bs.Marshal(u)
if err != nil {
return err
}
err = b.Put([]byte(u.Username), v)
if err != nil {
return err
}
}
return nil
})
}*/
func (db *DB) CreateUser(username, password string, admin bool) error {
username = userRe.ReplaceAllString(username, "")
if reservedUser(username) {
return errUserReserved
}

11
pkg/render/render.go
View file

@ -30,7 +30,7 @@ var (
func Render(p *core.Page) string {
rend := blackfriday.MarkdownCommon([]byte(p.Markdown))
index, html := buildIndex(string(rend))
index, html := buildTOC(string(rend))
html = strings.Replace(html, "<table>", `<table class="table">`, -1)
html = strings.Replace(html, "</h1>", `</h1><hr>`, -1)
html = strings.Replace(html, "<a", `<a target="_blank"`, -1)
@ -58,12 +58,17 @@ func UnstoreTitle(storeTitle string) (section, title string) {
return
}
func buildIndex(html string) (string, string) {
func buildTOC(html string) (string, string) {
var ret string
m := make(map[string]int)
hn := reIndex.FindAllStringSubmatch(html, -1)
html = reHeader.ReplaceAllString(html, `<a id="${2}" class="anchor"></a>${1}${2}${3}`)
html = reLink.ReplaceAllStringFunc(html, spaceReplace)
html = reLink.ReplaceAllStringFunc(html, func(s string) string {
if len(s) >= 3 {
return `id="` + Title(s[2:]) + `"`
}
return s
})
for k, v := range hn {
link := Title(v[2])
i := m[link]

62
pkg/server/handler.go
View file

@ -86,11 +86,11 @@ func loginHandler(ctx *Context) {
case "GET":
ctx.Exec()
case "POST":
user := ctx.Form("user")
password := ctx.Form("password")
if !ctx.CheckXsrf() {
return
}
user := ctx.Form("user")
password := ctx.Form("password")
if user == "" || password == "" {
ctx.Error("wrong inputs")
return
@ -115,10 +115,10 @@ func searchHandler(ctx *Context) {
case "GET":
ctx.Exec()
case "POST":
search := ctx.Form("search")
if !ctx.CheckXsrf() {
return
}
search := ctx.Form("search")
res, err := ctx.Srv.DB.Index.Search(search)
if err != nil {
@ -150,8 +150,11 @@ func pageNewHandler(ctx *Context) {
ctx.Data.Data = core.Page{Perm: core.Public}
ctx.Exec()
case "POST":
if !ctx.CheckXsrf() {
return
}
section := ctx.Form("section")
title := render.Title(ctx.Form("title"))
title := ctx.Form("title")
markdown := ctx.Form("markdown")
perm := core.ParsePermString(ctx.Form("perm"))
p := core.Page{
@ -160,9 +163,7 @@ func pageNewHandler(ctx *Context) {
Perm: perm,
}
ctx.Data.Data = p
if !ctx.CheckXsrf() {
return
}
if section != core.WikiSection {
section = ctx.User()
}
@ -213,7 +214,7 @@ func pageHandler(ctx *Context) {
ctx.Template("pageHandler")
}
}
page, err := ctx.Srv.DB.GetPage(render.StoreTitle(section, title), user)
page, err := ctx.Srv.DB.GetPage(section, title, user)
if err != nil {
log.Println(err)
ctx.NotFound()
@ -236,10 +237,9 @@ func pageShareHandler(ctx *Context) {
ctx.Template("pageShareHandler")
ctx.Data = webData{}
section := ctx.Var("section")
title := render.Title(ctx.Var("title"))
title := ctx.Var("title")
user := ctx.User()
st := render.StoreTitle(section, title)
page, err := ctx.Srv.DB.GetPage(st, user)
page, err := ctx.Srv.DB.GetPage(section, title, user)
if err != nil {
ctx.NotFound()
return
@ -248,18 +248,18 @@ func pageShareHandler(ctx *Context) {
ctx.Error("only private pages can be shared")
return
}
ctx.Data.Data = st
ctx.Data.Data = page.StoreTitle
ctx.Data.Title = page.Title
ctx.Data.BodyTitle = page.Title
switch ctx.Method() {
case "GET":
ctx.Exec()
case "POST":
duration := ctx.Form("duration")
if !ctx.CheckXsrf() {
return
}
d, err := time.ParseDuration(duration)
d, err := time.ParseDuration(ctx.Form("duration"))
if err != nil {
ctx.Error(err)
return
@ -295,6 +295,7 @@ func pageBlacklistHandler(ctx *Context) {
if !ctx.CheckXsrf() {
return
}
t, err := jwt.DecodeToken(ctx.Form("share"))
if err != nil {
ctx.Error(err)
@ -327,8 +328,8 @@ func pageEditHandler(ctx *Context) {
ctx.Template("pageEditHandler")
ctx.Data = webData{}
section := ctx.Var("section")
title := render.Title(ctx.Var("title"))
page, err := ctx.Srv.DB.GetPage(render.StoreTitle(section, title), ctx.User())
title := ctx.Var("title")
page, err := ctx.Srv.DB.GetPage(section, title, ctx.User())
if err != nil {
ctx.NotFound()
return
@ -340,12 +341,13 @@ func pageEditHandler(ctx *Context) {
ctx.Data.Data = page
ctx.Exec()
case "POST":
page.Markdown = ctx.Form("markdown")
page.Perm = core.ParsePermString(ctx.Form("perm"))
ctx.Data.Data = page
if !ctx.CheckXsrf() {
return
}
page.Markdown = ctx.Form("markdown")
page.Perm = core.ParsePermString(ctx.Form("perm"))
ctx.Data.Data = page
err = ctx.Srv.DB.UpdatePage(title, section, page.Markdown, ctx.User(), page.Perm)
if err != nil {
ctx.Error(err)
@ -357,10 +359,9 @@ func pageEditHandler(ctx *Context) {
func pageDelHandler(ctx *Context) {
section := ctx.Var("section")
title := render.Title(ctx.Var("title"))
st := render.StoreTitle(section, title)
title := ctx.Var("title")
ctx.Template("pageDelHandler")
ctx.Data.Data = st
ctx.Data.Data = render.StoreTitle(section, title)
ctx.Data.Title = title
ctx.Data.BodyTitle = title
switch ctx.Method() {
@ -407,6 +408,9 @@ func userNewHandler(ctx *Context) {
case "GET":
ctx.Exec()
case "POST":
if !ctx.CheckXsrf() {
return
}
user := ctx.Form("user")
password := ctx.Form("password")
repeat := ctx.Form("repeat")
@ -415,9 +419,7 @@ func userNewHandler(ctx *Context) {
if admin == "0" {
adm = true
}
if !ctx.CheckXsrf() {
return
}
if user == "" || password == "" || password != repeat {
ctx.Error("wrong inputs")
return
@ -446,6 +448,9 @@ func userEditHandler(ctx *Context) {
ctx.Data.Data = u
ctx.Exec()
case "POST":
if !ctx.CheckXsrf() {
return
}
user := ctx.Var("user")
password := ctx.Form("password")
repeat := ctx.Form("repeat")
@ -454,9 +459,7 @@ func userEditHandler(ctx *Context) {
if admin == "0" {
adm = true
}
if !ctx.CheckXsrf() {
return
}
if user == "" || password != repeat {
ctx.Error("wrong inputs")
return
@ -504,6 +507,9 @@ func userDelHandler(ctx *Context) {
ctx.Data.Data = user
ctx.Exec()
case "POST":
if !ctx.CheckXsrf() {
return
}
if err := ctx.Srv.DB.DeleteUser(ctx.Var("user")); err != nil {
log.Println("delete:", err)
}

10
pkg/server/routes.go
View file

@ -98,29 +98,29 @@ var routes = []route{
[]string{"GET"},
},
{
"/{section}/{title}",
"/{section:[a-zA-Z0-9]+}/{title:[a-zA-Z0-9+-]+}",
pageHandler,
[]string{"GET"},
},
{
"/{section}/{title}/md",
"/{section:[a-zA-Z0-9]+}/{title:[a-zA-Z0-9+-]+}/md",
pageMDHandler(
pageHandler),
[]string{"GET"},
},
{
"/{section}/{title}/share",
"/{section:[a-zA-Z0-9]+}/{title:[a-zA-Z0-9+-]+}/share",
pageShareHandler,
[]string{"GET", "POST"},
},
{
"/{section}/{title}/edit",
"/{section:[a-zA-Z0-9]+}/{title:[a-zA-Z0-9+-]+}/edit",
authHandler(
pageEditHandler),
[]string{"GET", "POST"},
},
{
"/{section}/{title}/del",
"/{section:[a-zA-Z0-9]+}/{title:[a-zA-Z0-9+-]+}/del",
authHandler(
pageDelHandler),
[]string{"GET", "POST"},

4
pkg/server/templates.go
View file

@ -50,7 +50,7 @@ const (
<link rel="stylesheet" type="text/css" href="/bootstrap.css" media="screen" integrity="sha256-NeLmQ7cX66J4MdtgGlG3O/TgvsSyP1b9WbaQtxYZUbQ="></link>
<link rel="stylesheet" type="text/css" href="/custom.css" media="screen" integrity="sha256-1LNAeuiM/rtsY1jIgLovUZ7HbYnOCkj3ZyailNA3rFQ="></link>
</head>
<body>
<body id="back-to-top">
<div class="navbar navbar-expand fixed-top navbar-dark bg-primary">
<div class="container">
<a href="/" class="navbar-brand">GoWiki</a>
@ -60,7 +60,7 @@ const (
</div>
</div>
<div class="container">
<div id="back-to-top" class="proper-content">
<div class="proper-content">
{{if .Msg}}
<div class="alert alert-danger msg">
<h4>Error!</h4>

10
pkg/store/boltstore.go
View file

@ -34,8 +34,8 @@ func NewBoltStore(file string, m Marshaler) (bs *BoltStore, err error) {
}
type dump struct {
k string `json:"k"`
v []byte `json:"v"`
K string `json:"k"`
V []byte `json:"v"`
}
func (bs *BoltStore) Dump(w io.Writer) (err error) {
@ -44,7 +44,9 @@ func (bs *BoltStore) Dump(w io.Writer) (err error) {
}
var d []dump
err = bs.ForEach(func(k string, v []byte) error {
d = append(d, dump{k, v})
b := make([]byte, len(v))
copy(b, v)
d = append(d, dump{k, b})
return nil
})
if err != nil {
@ -74,7 +76,7 @@ func (bs *BoltStore) Restore(r io.Reader) (err error) {
return bs.db.Update(func(tx *bolt.Tx) (err error) {
b := tx.Bucket([]byte(defaultBoltBucket))
for _, v := range d {
b.Put([]byte(v.k), v.v)
b.Put([]byte(v.K), v.V)
}
return nil
})

4
templates/index.html
View file

@ -5,7 +5,7 @@
<link rel="stylesheet" type="text/css" href="/bootstrap.css" media="screen" integrity="sha256-NeLmQ7cX66J4MdtgGlG3O/TgvsSyP1b9WbaQtxYZUbQ="></link>
<link rel="stylesheet" type="text/css" href="/custom.css" media="screen" integrity="sha256-1LNAeuiM/rtsY1jIgLovUZ7HbYnOCkj3ZyailNA3rFQ="></link>
</head>
<body>
<body id="back-to-top">
<div class="navbar navbar-expand fixed-top navbar-dark bg-primary">
<div class="container">
<a href="/" class="navbar-brand">GoWiki</a>
@ -15,7 +15,7 @@
</div>
</div>
<div class="container">
<div id="back-to-top" class="proper-content">
<div class="proper-content">
{{if .Msg}}
<div class="alert alert-danger msg">
<h4>Error!</h4>

302
vendor/github.com/blevesearch/segment/segment_words.go generated vendored
View file

@ -1,4 +1,3 @@
//line segment_words.rl:1
// Copyright (c) 2015 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
@ -31,7 +30,6 @@ const (
Ideo
)
//line segment_words.go:36
var _s_key_offsets []uint16 = []uint16{
0, 1, 3, 5, 7, 10, 15, 20,
@ -18539,10 +18537,8 @@ const s_error int = -1
const s_en_main int = 4862
//line segment_words.rl:35
func segmentWords(data []byte, maxTokens int, atEOF bool, val [][]byte, types []int) ([][]byte, []int, int, error) {
cs, p, pe := 0, 0, len(data)
cap := maxTokens
@ -18587,12 +18583,11 @@ func segmentWords(data []byte, maxTokens int, atEOF bool, val [][]byte, types []
if p == pe {
goto _test_eof
}
_resume:
_resume:
switch _s_from_state_actions[cs] {
case 23:
//line NONE:1
ts = p
ts = p
//line segment_words.go:18598
}
@ -18639,7 +18634,7 @@ ts = p
switch {
case data[p] < _s_trans_keys[_mid]:
_upper = _mid - 2
case data[p] > _s_trans_keys[_mid + 1]:
case data[p] > _s_trans_keys[_mid+1]:
_lower = _mid + 2
default:
_trans += int((_mid - int(_keys)) >> 1)
@ -18649,9 +18644,9 @@ ts = p
_trans += _klen
}
_match:
_match:
_trans = int(_s_indicies[_trans])
_eof_trans:
_eof_trans:
cs = int(_s_trans_targs[_trans])
if _s_trans_actions[_trans] == 0 {
@ -18664,19 +18659,18 @@ _eof_trans:
endPos = p
case 34:
//line segment_words.rl:76
te = p
p--
{
te = p
p--
{
if !atEOF {
return val, types, totalConsumed, nil
}
val = append(val, data[startPos:endPos+1])
types = append(types, Number)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -18684,9 +18678,9 @@ p--
case 37:
//line segment_words.rl:89
te = p
p--
{
te = p
p--
{
if endPos+1 == pe && !atEOF {
return val, types, totalConsumed, nil
} else if dr, size := utf8.DecodeRune(data[endPos+1:]); dr == utf8.RuneError && size == 1 {
@ -18695,7 +18689,7 @@ p--
val = append(val, data[startPos:endPos+1])
types = append(types, Letter)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -18703,9 +18697,9 @@ p--
case 39:
//line segment_words.rl:104
te = p
p--
{
te = p
p--
{
if endPos+1 == pe && !atEOF {
return val, types, totalConsumed, nil
} else if dr, size := utf8.DecodeRune(data[endPos+1:]); dr == utf8.RuneError && size == 1 {
@ -18714,7 +18708,7 @@ p--
val = append(val, data[startPos:endPos+1])
types = append(types, Ideo)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -18722,15 +18716,15 @@ p--
case 35:
//line segment_words.rl:119
te = p
p--
{
te = p
p--
{
if !atEOF {
return val, types, totalConsumed, nil
}
val = append(val, data[startPos:endPos+1])
types = append(types, Letter)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -18738,9 +18732,9 @@ p--
case 38:
//line segment_words.rl:131
te = p
p--
{
te = p
p--
{
if endPos+1 == pe && !atEOF {
return val, types, totalConsumed, nil
} else if dr, size := utf8.DecodeRune(data[endPos+1:]); dr == utf8.RuneError && size == 1 {
@ -18749,7 +18743,7 @@ p--
val = append(val, data[startPos:endPos+1])
types = append(types, Ideo)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -18757,9 +18751,9 @@ p--
case 40:
//line segment_words.rl:146
te = p
p--
{
te = p
p--
{
if endPos+1 == pe && !atEOF {
return val, types, totalConsumed, nil
} else if dr, size := utf8.DecodeRune(data[endPos+1:]); dr == utf8.RuneError && size == 1 {
@ -18768,7 +18762,7 @@ p--
val = append(val, data[startPos:endPos+1])
types = append(types, Ideo)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -18776,15 +18770,15 @@ p--
case 41:
//line segment_words.rl:161
te = p
p--
{
te = p
p--
{
lastPos := startPos
for lastPos <= endPos {
_, size := utf8.DecodeRune(data[lastPos:])
lastPos += size
}
endPos = lastPos -1
endPos = lastPos - 1
p = endPos
if endPos+1 == pe && !atEOF {
@ -18795,7 +18789,7 @@ p--
// otherwise, consume this as well
val = append(val, data[startPos:endPos+1])
types = append(types, None)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -18803,15 +18797,15 @@ p--
case 32:
//line segment_words.rl:161
te = p
p--
{
te = p
p--
{
lastPos := startPos
for lastPos <= endPos {
_, size := utf8.DecodeRune(data[lastPos:])
lastPos += size
}
endPos = lastPos -1
endPos = lastPos - 1
p = endPos
if endPos+1 == pe && !atEOF {
@ -18822,7 +18816,7 @@ p--
// otherwise, consume this as well
val = append(val, data[startPos:endPos+1])
types = append(types, None)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -18830,15 +18824,15 @@ p--
case 36:
//line segment_words.rl:161
te = p
p--
{
te = p
p--
{
lastPos := startPos
for lastPos <= endPos {
_, size := utf8.DecodeRune(data[lastPos:])
lastPos += size
}
endPos = lastPos -1
endPos = lastPos - 1
p = endPos
if endPos+1 == pe && !atEOF {
@ -18849,7 +18843,7 @@ p--
// otherwise, consume this as well
val = append(val, data[startPos:endPos+1])
types = append(types, None)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -18857,15 +18851,15 @@ p--
case 31:
//line segment_words.rl:161
te = p
p--
{
te = p
p--
{
lastPos := startPos
for lastPos <= endPos {
_, size := utf8.DecodeRune(data[lastPos:])
lastPos += size
}
endPos = lastPos -1
endPos = lastPos - 1
p = endPos
if endPos+1 == pe && !atEOF {
@ -18876,7 +18870,7 @@ p--
// otherwise, consume this as well
val = append(val, data[startPos:endPos+1])
types = append(types, None)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -18884,15 +18878,15 @@ p--
case 4:
//line segment_words.rl:76
p = (te) - 1
{
p = (te) - 1
{
if !atEOF {
return val, types, totalConsumed, nil
}
val = append(val, data[startPos:endPos+1])
types = append(types, Number)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -18900,8 +18894,8 @@ p = (te) - 1
case 12:
//line segment_words.rl:89
p = (te) - 1
{
p = (te) - 1
{
if endPos+1 == pe && !atEOF {
return val, types, totalConsumed, nil
} else if dr, size := utf8.DecodeRune(data[endPos+1:]); dr == utf8.RuneError && size == 1 {
@ -18910,7 +18904,7 @@ p = (te) - 1
val = append(val, data[startPos:endPos+1])
types = append(types, Letter)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -18918,8 +18912,8 @@ p = (te) - 1
case 17:
//line segment_words.rl:104
p = (te) - 1
{
p = (te) - 1
{
if endPos+1 == pe && !atEOF {
return val, types, totalConsumed, nil
} else if dr, size := utf8.DecodeRune(data[endPos+1:]); dr == utf8.RuneError && size == 1 {
@ -18928,7 +18922,7 @@ p = (te) - 1
val = append(val, data[startPos:endPos+1])
types = append(types, Ideo)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -18936,14 +18930,14 @@ p = (te) - 1
case 7:
//line segment_words.rl:119
p = (te) - 1
{
p = (te) - 1
{
if !atEOF {
return val, types, totalConsumed, nil
}
val = append(val, data[startPos:endPos+1])
types = append(types, Letter)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -18951,8 +18945,8 @@ p = (te) - 1
case 15:
//line segment_words.rl:131
p = (te) - 1
{
p = (te) - 1
{
if endPos+1 == pe && !atEOF {
return val, types, totalConsumed, nil
} else if dr, size := utf8.DecodeRune(data[endPos+1:]); dr == utf8.RuneError && size == 1 {
@ -18961,7 +18955,7 @@ p = (te) - 1
val = append(val, data[startPos:endPos+1])
types = append(types, Ideo)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -18969,8 +18963,8 @@ p = (te) - 1
case 19:
//line segment_words.rl:146
p = (te) - 1
{
p = (te) - 1
{
if endPos+1 == pe && !atEOF {
return val, types, totalConsumed, nil
} else if dr, size := utf8.DecodeRune(data[endPos+1:]); dr == utf8.RuneError && size == 1 {
@ -18979,7 +18973,7 @@ p = (te) - 1
val = append(val, data[startPos:endPos+1])
types = append(types, Ideo)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -18987,14 +18981,14 @@ p = (te) - 1
case 21:
//line segment_words.rl:161
p = (te) - 1
{
p = (te) - 1
{
lastPos := startPos
for lastPos <= endPos {
_, size := utf8.DecodeRune(data[lastPos:])
lastPos += size
}
endPos = lastPos -1
endPos = lastPos - 1
p = endPos
if endPos+1 == pe && !atEOF {
@ -19005,7 +18999,7 @@ p = (te) - 1
// otherwise, consume this as well
val = append(val, data[startPos:endPos+1])
types = append(types, None)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -19013,14 +19007,14 @@ p = (te) - 1
case 8:
//line segment_words.rl:161
p = (te) - 1
{
p = (te) - 1
{
lastPos := startPos
for lastPos <= endPos {
_, size := utf8.DecodeRune(data[lastPos:])
lastPos += size
}
endPos = lastPos -1
endPos = lastPos - 1
p = endPos
if endPos+1 == pe && !atEOF {
@ -19031,7 +19025,7 @@ p = (te) - 1
// otherwise, consume this as well
val = append(val, data[startPos:endPos+1])
types = append(types, None)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -19039,14 +19033,14 @@ p = (te) - 1
case 1:
//line segment_words.rl:161
p = (te) - 1
{
p = (te) - 1
{
lastPos := startPos
for lastPos <= endPos {
_, size := utf8.DecodeRune(data[lastPos:])
lastPos += size
}
endPos = lastPos -1
endPos = lastPos - 1
p = endPos
if endPos+1 == pe && !atEOF {
@ -19057,7 +19051,7 @@ p = (te) - 1
// otherwise, consume this as well
val = append(val, data[startPos:endPos+1])
types = append(types, None)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -19067,7 +19061,8 @@ p = (te) - 1
//line NONE:1
switch act {
case 1:
{p = (te) - 1
{
p = (te) - 1
if !atEOF {
return val, types, totalConsumed, nil
@ -19075,13 +19070,14 @@ p = (te) - 1
val = append(val, data[startPos:endPos+1])
types = append(types, Number)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
}
case 2:
{p = (te) - 1
{
p = (te) - 1
if endPos+1 == pe && !atEOF {
return val, types, totalConsumed, nil
@ -19091,13 +19087,14 @@ p = (te) - 1
val = append(val, data[startPos:endPos+1])
types = append(types, Letter)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
}
case 3:
{p = (te) - 1
{
p = (te) - 1
if endPos+1 == pe && !atEOF {
return val, types, totalConsumed, nil
@ -19107,26 +19104,28 @@ p = (te) - 1
val = append(val, data[startPos:endPos+1])
types = append(types, Ideo)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
}
case 4:
{p = (te) - 1
{
p = (te) - 1
if !atEOF {
return val, types, totalConsumed, nil
}
val = append(val, data[startPos:endPos+1])
types = append(types, Letter)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
}
case 5:
{p = (te) - 1
{
p = (te) - 1
if endPos+1 == pe && !atEOF {
return val, types, totalConsumed, nil
@ -19136,20 +19135,21 @@ p = (te) - 1
val = append(val, data[startPos:endPos+1])
types = append(types, Ideo)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
}
case 7:
{p = (te) - 1
{
p = (te) - 1
lastPos := startPos
for lastPos <= endPos {
_, size := utf8.DecodeRune(data[lastPos:])
lastPos += size
}
endPos = lastPos -1
endPos = lastPos - 1
p = endPos
if endPos+1 == pe && !atEOF {
@ -19160,20 +19160,21 @@ p = (te) - 1
// otherwise, consume this as well
val = append(val, data[startPos:endPos+1])
types = append(types, None)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
}
case 12:
{p = (te) - 1
{
p = (te) - 1
lastPos := startPos
for lastPos <= endPos {
_, size := utf8.DecodeRune(data[lastPos:])
lastPos += size
}
endPos = lastPos -1
endPos = lastPos - 1
p = endPos
if endPos+1 == pe && !atEOF {
@ -19184,20 +19185,21 @@ p = (te) - 1
// otherwise, consume this as well
val = append(val, data[startPos:endPos+1])
types = append(types, None)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
}
case 13:
{p = (te) - 1
{
p = (te) - 1
lastPos := startPos
for lastPos <= endPos {
_, size := utf8.DecodeRune(data[lastPos:])
lastPos += size
}
endPos = lastPos -1
endPos = lastPos - 1
p = endPos
if endPos+1 == pe && !atEOF {
@ -19208,14 +19210,13 @@ p = (te) - 1
// otherwise, consume this as well
val = append(val, data[startPos:endPos+1])
types = append(types, None)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
}
}
case 28:
//line segment_words.rl:68
@ -19225,21 +19226,20 @@ p = (te) - 1
endPos = p
case 33:
//line segment_words.rl:72
endPos = p
//line segment_words.rl:161
te = p+1
{
te = p + 1
{
lastPos := startPos
for lastPos <= endPos {
_, size := utf8.DecodeRune(data[lastPos:])
lastPos += size
}
endPos = lastPos -1
endPos = lastPos - 1
p = endPos
if endPos+1 == pe && !atEOF {
@ -19250,7 +19250,7 @@ te = p+1
// otherwise, consume this as well
val = append(val, data[startPos:endPos+1])
types = append(types, None)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -19262,14 +19262,14 @@ te = p+1
endPos = p
//line segment_words.rl:161
te = p+1
{
te = p + 1
{
lastPos := startPos
for lastPos <= endPos {
_, size := utf8.DecodeRune(data[lastPos:])
lastPos += size
}
endPos = lastPos -1
endPos = lastPos - 1
p = endPos
if endPos+1 == pe && !atEOF {
@ -19280,7 +19280,7 @@ te = p+1
// otherwise, consume this as well
val = append(val, data[startPos:endPos+1])
types = append(types, None)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -19288,13 +19288,12 @@ te = p+1
case 18:
//line NONE:1
te = p+1
te = p + 1
//line segment_words.rl:72
endPos = p
case 26:
//line segment_words.rl:68
@ -19305,14 +19304,14 @@ te = p+1
endPos = p
//line segment_words.rl:161
te = p+1
{
te = p + 1
{
lastPos := startPos
for lastPos <= endPos {
_, size := utf8.DecodeRune(data[lastPos:])
lastPos += size
}
endPos = lastPos -1
endPos = lastPos - 1
p = endPos
if endPos+1 == pe && !atEOF {
@ -19323,7 +19322,7 @@ te = p+1
// otherwise, consume this as well
val = append(val, data[startPos:endPos+1])
types = append(types, None)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -19339,14 +19338,14 @@ te = p+1
endPos = p
//line segment_words.rl:161
te = p+1
{
te = p + 1
{
lastPos := startPos
for lastPos <= endPos {
_, size := utf8.DecodeRune(data[lastPos:])
lastPos += size
}
endPos = lastPos -1
endPos = lastPos - 1
p = endPos
if endPos+1 == pe && !atEOF {
@ -19357,7 +19356,7 @@ te = p+1
// otherwise, consume this as well
val = append(val, data[startPos:endPos+1])
types = append(types, None)
totalConsumed = endPos+1
totalConsumed = endPos + 1
if maxTokens > 0 && len(val) >= maxTokens {
return val, types, totalConsumed, nil
}
@ -19365,95 +19364,95 @@ te = p+1
case 5:
//line NONE:1
te = p+1
te = p + 1
//line segment_words.rl:72
endPos = p
//line segment_words.rl:76
act = 1;
act = 1
case 11:
//line NONE:1
te = p+1
te = p + 1
//line segment_words.rl:72
endPos = p
//line segment_words.rl:89
act = 2;
act = 2
case 16:
//line NONE:1
te = p+1
te = p + 1
//line segment_words.rl:72
endPos = p
//line segment_words.rl:104
act = 3;
act = 3
case 6:
//line NONE:1
te = p+1
te = p + 1
//line segment_words.rl:72
endPos = p
//line segment_words.rl:119
act = 4;
act = 4
case 14:
//line NONE:1
te = p+1
te = p + 1
//line segment_words.rl:72
endPos = p
//line segment_words.rl:131
act = 5;
act = 5
case 20:
//line NONE:1
te = p+1
te = p + 1
//line segment_words.rl:72
endPos = p
//line segment_words.rl:161
act = 7;
act = 7
case 9:
//line NONE:1
te = p+1
te = p + 1
//line segment_words.rl:72
endPos = p
//line segment_words.rl:161
act = 12;
act = 12
case 2:
//line NONE:1
te = p+1
te = p + 1
//line segment_words.rl:72
endPos = p
//line segment_words.rl:161
act = 13;
act = 13
case 29:
//line NONE:1
te = p+1
te = p + 1
//line segment_words.rl:68
@ -19464,11 +19463,11 @@ te = p+1
endPos = p
//line segment_words.rl:76
act = 1;
act = 1
case 30:
//line NONE:1
te = p+1
te = p + 1
//line segment_words.rl:68
@ -19479,11 +19478,11 @@ te = p+1
endPos = p
//line segment_words.rl:119
act = 4;
act = 4
case 25:
//line NONE:1
te = p+1
te = p + 1
//line segment_words.rl:68
@ -19494,17 +19493,16 @@ te = p+1
endPos = p
//line segment_words.rl:161
act = 13;
act = 13
//line segment_words.go:19500
}
_again:
_again:
switch _s_to_state_actions[cs] {
case 22:
//line NONE:1
ts = 0
ts = 0
//line segment_words.go:19510
}
@ -19512,7 +19510,9 @@ ts = 0
if p++; p != pe {
goto _resume
}
_test_eof: {}
_test_eof:
{
}
if p == eof {
if _s_eof_trans[cs] > 0 {
_trans = int(_s_eof_trans[cs] - 1)
@ -19524,7 +19524,6 @@ ts = 0
startPos = p
//line segment_words.go:19529
}
}
@ -19533,7 +19532,6 @@ ts = 0
//line segment_words.rl:278
if cs < s_first_final {
return val, types, totalConsumed, ParseError
}

24417
vendor/github.com/blevesearch/segment/segment_words_prod.go generated vendored
View file

File diff suppressed because it is too large Load diff