added text indexing

pkg/scan/helper.go

@@ -6,6 +6,7 @@ import (
 	"bytes"
 	"net/http"
 	"os"
+	"strings"
 )
 
 const (
@@ -18,6 +19,8 @@ const (
 	pnmT         = "image/x-portable-anymap"
 	jp2T         = "image/jp2"
 	tiffT        = "image/tiff"
+	textT        = "text/"
+	rawT         = "application/octet-stream"
 	maxSniffSize = 512
 )
 
@@ -45,35 +48,11 @@ func extendedContentType(buf []byte) string {
 			return sig.ct
 		}
 	}
-	return "application/octet-stream"
+	return rawT
 }
 
-func isPDF(path string) bool {
-	f, err := os.Open(path)
-	if err != nil {
-		return false
-	}
-	defer f.Close()
-	buf := make([]byte, maxSniffSize)
-	_, err = f.Read(buf)
-	if err != nil {
-		return false
-	}
-	return http.DetectContentType(buf) == pdfT
-}
-
-func isImage(path string) bool {
-	f, err := os.Open(path)
-	if err != nil {
-		return false
-	}
-	defer f.Close()
-	buf := make([]byte, maxSniffSize)
-	_, err = f.Read(buf)
-	if err != nil {
-		return false
-	}
-	switch http.DetectContentType(buf) {
+func isImage(ct string, buf []byte) bool {
+	switch ct {
 	case jpegT, pngT, bmpT, gifT, webpT:
 		return true
 	default:
@@ -84,3 +63,29 @@ func isImage(path string) bool {
 	}
 	return false
 }
+
+type filetype struct {
+	text  bool
+	pdf   bool
+	image bool
+}
+
+func (s *Scanner) newFileType(path string) (ft *filetype) {
+	ft = new(filetype)
+	f, err := os.Open(path)
+	if err != nil {
+		return
+	}
+	defer f.Close()
+	buf := make([]byte, maxSniffSize)
+	n, err := f.Read(buf)
+	if err != nil {
+		return
+	}
+	buf = buf[:n]
+	ct := http.DetectContentType(buf)
+	ft.text = strings.HasPrefix(ct, textT)
+	ft.pdf = ct == pdfT
+	ft.image = isImage(ct, buf)
+	return
+}