diff --git a/pkg/scan/helper.go b/pkg/scan/helper.go index dd5259c..eee8703 100644 --- a/pkg/scan/helper.go +++ b/pkg/scan/helper.go @@ -6,6 +6,7 @@ import ( "bytes" "net/http" "os" + "strings" ) const ( @@ -18,6 +19,8 @@ const ( pnmT = "image/x-portable-anymap" jp2T = "image/jp2" tiffT = "image/tiff" + textT = "text/" + rawT = "application/octet-stream" maxSniffSize = 512 ) @@ -45,35 +48,11 @@ func extendedContentType(buf []byte) string { return sig.ct } } - return "application/octet-stream" + return rawT } -func isPDF(path string) bool { - f, err := os.Open(path) - if err != nil { - return false - } - defer f.Close() - buf := make([]byte, maxSniffSize) - _, err = f.Read(buf) - if err != nil { - return false - } - return http.DetectContentType(buf) == pdfT -} - -func isImage(path string) bool { - f, err := os.Open(path) - if err != nil { - return false - } - defer f.Close() - buf := make([]byte, maxSniffSize) - _, err = f.Read(buf) - if err != nil { - return false - } - switch http.DetectContentType(buf) { +func isImage(ct string, buf []byte) bool { + switch ct { case jpegT, pngT, bmpT, gifT, webpT: return true default: @@ -84,3 +63,29 @@ func isImage(path string) bool { } return false } + +type filetype struct { + text bool + pdf bool + image bool +} + +func (s *Scanner) newFileType(path string) (ft *filetype) { + ft = new(filetype) + f, err := os.Open(path) + if err != nil { + return + } + defer f.Close() + buf := make([]byte, maxSniffSize) + n, err := f.Read(buf) + if err != nil { + return + } + buf = buf[:n] + ct := http.DetectContentType(buf) + ft.text = strings.HasPrefix(ct, textT) + ft.pdf = ct == pdfT + ft.image = isImage(ct, buf) + return +} diff --git a/pkg/scan/scanner.go b/pkg/scan/scanner.go index 85663fa..3ea27d7 100644 --- a/pkg/scan/scanner.go +++ b/pkg/scan/scanner.go @@ -108,11 +108,23 @@ func (s *Scanner) Scan(file string) (filename, text string, err error) { file = fs.Clean(file) scanfile := filepath.Join(s.base, file) var pdffile string - pdf := isPDF(scanfile) - if !pdf { - if !isImage(scanfile) { - err = errors.New("error: input file is not an image") - return + ft := s.newFileType(scanfile) + if !ft.pdf { + if !ft.image { + if ft.text { + log.V(2).Info("indexing plaintext file", "file", scanfile) + var buf []byte + buf, err = os.ReadFile(scanfile) + if err != nil { + return + } + filename = file + text = string(nlr.ReplaceAll(wsr.ReplaceAll(buf, ws), nl)) + return + } else { + err = errors.New("error: input: no image, pdf or text file") + return + } } var ( pdf []byte @@ -154,7 +166,7 @@ func (s *Scanner) Scan(file string) (filename, text string, err error) { return } text = string(nlr.ReplaceAll(wsr.ReplaceAll(txt, ws), nl)) - if !pdf { + if !ft.pdf { err = os.Remove(scanfile) } return