added text indexing

pkg/scan/helper.go

@@ -6,6 +6,7 @@ import (
 	"bytes"
 	"net/http"
 	"os"
+	"strings"
 )
 
 const (
@@ -18,6 +19,8 @@ const (
 	pnmT         = "image/x-portable-anymap"
 	jp2T         = "image/jp2"
 	tiffT        = "image/tiff"
+	textT        = "text/"
+	rawT         = "application/octet-stream"
 	maxSniffSize = 512
 )
 
@@ -45,35 +48,11 @@ func extendedContentType(buf []byte) string {
 			return sig.ct
 		}
 	}
-	return "application/octet-stream"
+	return rawT
 }
 
-func isPDF(path string) bool {
+func isImage(ct string, buf []byte) bool {
-	f, err := os.Open(path)
+	switch ct {
-	if err != nil {
-		return false
-	}
-	defer f.Close()
-	buf := make([]byte, maxSniffSize)
-	_, err = f.Read(buf)
-	if err != nil {
-		return false
-	}
-	return http.DetectContentType(buf) == pdfT
-}
-
-func isImage(path string) bool {
-	f, err := os.Open(path)
-	if err != nil {
-		return false
-	}
-	defer f.Close()
-	buf := make([]byte, maxSniffSize)
-	_, err = f.Read(buf)
-	if err != nil {
-		return false
-	}
-	switch http.DetectContentType(buf) {
 	case jpegT, pngT, bmpT, gifT, webpT:
 		return true
 	default:
@@ -84,3 +63,29 @@ func isImage(path string) bool {
 	}
 	return false
 }
+
+type filetype struct {
+	text  bool
+	pdf   bool
+	image bool
+}
+
+func (s *Scanner) newFileType(path string) (ft *filetype) {
+	ft = new(filetype)
+	f, err := os.Open(path)
+	if err != nil {
+		return
+	}
+	defer f.Close()
+	buf := make([]byte, maxSniffSize)
+	n, err := f.Read(buf)
+	if err != nil {
+		return
+	}
+	buf = buf[:n]
+	ct := http.DetectContentType(buf)
+	ft.text = strings.HasPrefix(ct, textT)
+	ft.pdf = ct == pdfT
+	ft.image = isImage(ct, buf)
+	return
+}

pkg/scan/scanner.go

@@ -108,12 +108,24 @@ func (s *Scanner) Scan(file string) (filename, text string, err error) {
 	file = fs.Clean(file)
 	scanfile := filepath.Join(s.base, file)
 	var pdffile string
-	pdf := isPDF(scanfile)
+	ft := s.newFileType(scanfile)
-	if !pdf {
+	if !ft.pdf {
-		if !isImage(scanfile) {
+		if !ft.image {
-			err = errors.New("error: input file is not an image")
+			if ft.text {
+				log.V(2).Info("indexing plaintext file", "file", scanfile)
+				var buf []byte
+				buf, err = os.ReadFile(scanfile)
+				if err != nil {
 					return
 				}
+				filename = file
+				text = string(nlr.ReplaceAll(wsr.ReplaceAll(buf, ws), nl))
+				return
+			} else {
+				err = errors.New("error: input: no image, pdf or text file")
+				return
+			}
+		}
 		var (
 			pdf []byte
 			f   *os.File
@@ -154,7 +166,7 @@ func (s *Scanner) Scan(file string) (filename, text string, err error) {
 		return
 	}
 	text = string(nlr.ReplaceAll(wsr.ReplaceAll(txt, ws), nl))
-	if !pdf {
+	if !ft.pdf {
 		err = os.Remove(scanfile)
 	}
 	return