better jobs queue

pkg/server/handler.go

@@ -166,7 +166,7 @@ func indexHandler(ctx *Context) {
 		if fpath, ok = hasTrimPrefix(path, core.IndexPrefix); ok {
 			log = log.WithValues("file", fpath)
 			log.V(2).Info("started indexing file")
-			err = ctx.Srv.SQ.Add(fpath)
+			err = ctx.Srv.SQ.Add(fpath, nil)
 			if err != nil {
 				log.Error(err, "error indexing file")
 			}
@@ -214,12 +214,11 @@ func indexHandler(ctx *Context) {
 				ctx.Srv.DB.IndexMutex.Set(uint64(len(files)))
 				for _, f := range files {
 					if !ctx.Srv.DB.IsIndexed(f) {
-						err := ctx.Srv.SQ.Add(f)
+						err := ctx.Srv.SQ.Add(f, ctx.Srv.DB.IndexMutex.Inc)
 						if err != nil {
 							log.Error(err, "error indexing file", "file", f)
 						}
 					}
-					ctx.Srv.DB.IndexMutex.Inc()
 				}
 			}()
 		} else {

pkg/server/queue.go

@@ -5,20 +5,25 @@ package server
 import "context"
 
 type ScanQueue struct {
-	ctx   context.Context
-	srv   *HTTPServer
-	pchan chan string
+	ctx  context.Context
+	srv  *HTTPServer
+	jobs chan job
+}
+
+type job struct {
+	path     string
+	callback func()
 }
 
 func NewScanQueue(ctx context.Context, srv *HTTPServer, buffer int) *ScanQueue {
 	return &ScanQueue{
-		ctx:   ctx,
-		srv:   srv,
-		pchan: make(chan string, buffer),
+		ctx:  ctx,
+		srv:  srv,
+		jobs: make(chan job, buffer),
 	}
 }
 
-func (sq *ScanQueue) Add(path string) (err error) {
+func (sq *ScanQueue) Add(path string, callback func()) (err error) {
 	err = sq.srv.FS.AddScan(path)
 	if err != nil {
 		return
@@ -27,7 +32,7 @@ func (sq *ScanQueue) Add(path string) (err error) {
 		select {
 		case <-sq.ctx.Done():
 			return
-		case sq.pchan <- path:
+		case sq.jobs <- job{path, callback}:
 		}
 	}()
 	return
@@ -37,16 +42,20 @@ func (sq *ScanQueue) Scan() {
 	for {
 		select {
 		case <-sq.ctx.Done():
-			//close(sq.pchan)
+			//close(sq.jobs)
 			return
-		case p := <-sq.pchan:
-			sq.scanFile(p)
+		case j := <-sq.jobs:
+			sq.scanFile(j)
 		}
 	}
 }
 
-func (sq *ScanQueue) scanFile(path string) {
+func (sq *ScanQueue) scanFile(j job) {
 	srv := sq.srv
+	path := j.path
+	if j.callback != nil {
+		defer j.callback()
+	}
 	defer srv.FS.RemoveScan(path)
 	log := srv.Log.WithValues("file", path)
 	file, txt, err := srv.Scanner.Scan(path)