mjhjmtu

I have thousand of documents and some of them are scanned. So I need a script to test all pdf files that belong to a directory. Is there a simple way to do that?

1. Most pdfs are reports. Thus they have a lot of text.


2. 
   

   They are very different, but the scanned ones as mentioned below one can find some text due to a precarious ocr process coupled to the scan.

   
   
   
   • NotScanned
   
   
   • Scanned1
   
   
   • Scanned2
   
   



3. The proposal due to Sudodus in the comments below seems to be very interesting. Look at the difference between a scanned to a not scanned-pdf:

Scanned:

grep --color -a 'Image' AR-G1002.pdf
<</BitsPerComponent 8/ColorSpace/DeviceRGB/Filter[/DCTDecode]/Height 2197/Length 340615/Name/Obj13/Subtype/Image/Type/XObject/Width 1698>>stream
<</BitsPerComponent 1/ColorSpace/DeviceGray/DecodeParms<</Columns 1698/K -1>>/Filter/CCITTFaxDecode/Height 2197/Length 40452/Name/Obj18/Subtype/Image/Type/XObject/Width 1698>>stream
<</BitsPerComponent 1/ColorSpace/DeviceGray/DecodeParms<</Columns 1698/K -1>>/Filter/CCITTFaxDecode/Height 2197/Length 41680/Name/Obj23/Subtype/Image/Type/XObject/Width 1698>>stream
<</BitsPerComponent 1/ColorSpace/DeviceGray/DecodeParms<</Columns 1698/K -1>>/Filter/CCITTFaxDecode/Height 2197/Length 41432/Name/Obj28/Subtype/Image/Type/XObject/Width 1698>>stream
<</BitsPerComponent 1/ColorSpace/DeviceGray/DecodeParms<</Columns 1698/K -1>>/Filter/CCITTFaxDecode/Height 2197/Length 59084/Name/Obj33/Subtype/Image/Type/XObject/Width 1698>>stream
<</BitsPerComponent 8/ColorSpace/DeviceRGB/Filter[/DCTDecode]/Height 2197/Length 472681/Name/Obj38/Subtype/Image/Type/XObject/Width 1698>>stream
<</BitsPerComponent 8/ColorSpace/DeviceRGB/Filter[/DCTDecode]/Height 2197/Length 469340/Name/Obj43/Subtype/Image/Type/XObject/Width 1698>>stream
<</BitsPerComponent 8/ColorSpace/DeviceRGB/Filter[/DCTDecode]/Height 2197/Length 371863/Name/Obj48/Subtype/Image/Type/XObject/Width 1698>>stream
<</BitsPerComponent 8/ColorSpace/DeviceRGB/Filter[/DCTDecode]/Height 2197/Length 344092/Name/Obj53/Subtype/Image/Type/XObject/Width 1698>>stream
<</BitsPerComponent 1/ColorSpace/DeviceGray/DecodeParms<</Columns 1698/K -1>>/Filter/CCITTFaxDecode/Height 2197/Length 59416/Name/Obj58/Subtype/Image/Type/XObject/Width 1698>>stream
<</BitsPerComponent 1/ColorSpace/DeviceGray/DecodeParms<</Columns 1698/K -1>>/Filter/CCITTFaxDecode/Height 2197/Length 48308/Name/Obj63/Subtype/Image/Type/XObject/Width 1698>>stream
<</BitsPerComponent 1/ColorSpace/DeviceGray/DecodeParms<</Columns 1698/K -1>>/Filter/CCITTFaxDecode/Height 2197/Length 51564/Name/Obj68/Subtype/Image/Type/XObject/Width 1698>>stream
<</BitsPerComponent 1/ColorSpace/DeviceGray/DecodeParms<</Columns 1698/K -1>>/Filter/CCITTFaxDecode/Height 2197/Length 63184/Name/Obj73/Subtype/Image/Type/XObject/Width 1698>>stream
<</BitsPerComponent 1/ColorSpace/DeviceGray/DecodeParms<</Columns 1698/K -1>>/Filter/CCITTFaxDecode/Height 2197/Length 40824/Name/Obj78/Subtype/Image/Type/XObject/Width 1698>>stream
<</BitsPerComponent 1/ColorSpace/DeviceGray/DecodeParms<</Columns 1698/K -1>>/Filter/CCITTFaxDecode/Height 2197/Length 23320/Name/Obj83/Subtype/Image/Type/XObject/Width 1698>>stream
<</BitsPerComponent 1/ColorSpace/DeviceGray/DecodeParms<</Columns 1698/K -1>>/Filter/CCITTFaxDecode/Height 2197/Length 31504/Name/Obj93/Subtype/Image/Type/XObject/Width 1698>>stream
<</BitsPerComponent 1/ColorSpace/DeviceGray/DecodeParms<</Columns 1698/K -1>>/Filter/CCITTFaxDecode/Height 2197/Length 18996/Name/Obj98/Subtype/Image/Type/XObject/Width 1698>>stream
<</BitsPerComponent 8/ColorSpace/DeviceRGB/Filter[/DCTDecode]/Height 2197/Length 292932/Name/Obj103/Subtype/Image/Type/XObject/Width 1698>>stream
<</BitsPerComponent 1/ColorSpace/DeviceGray/DecodeParms<</Columns 1698/K -1>>/Filter/CCITTFaxDecode/Height 2197/Length 27720/Name/Obj108/Subtype/Image/Type/XObject/Width 1698>>stream
 <rdf:li xml:lang="x-default">Image</rdf:li>
 <rdf:li xml:lang="x-default">Image</rdf:li>

Not Scanned:

grep --color -a 'Image' AR-G1003.pdf
<</Lang(en-US)/MarkInfo<</Marked true>>/Metadata 167 0 R/Pages 2 0 R/StructTreeR<</Contents 4 0 R/Group<</CS/DeviceRGB/S/Transparency/Type/Group>>/MediaBox[0 0 612 792]/Parent 2 0 R/Resources<</Font<</F1 5 0 R/F2 7 0 R/F3 9 0 R/F4 11 0 R/F5 13 0 R>>/ProcSet[/PDF/Text/ImageB/ImageC/ImageI]>>/StructParents 0/Tabs/S/Type/<</Filter/FlateDecode/Length 5463>>stream
<</BaseFont/Times#20New#20Roman,Bold/Encoding/WinAnsiEncoding/FirstChar 32/FontD<</Ascent 891/AvgWidth 427/CapHeight 677/Descent -216/Flags 32/FontBBox[-558 -216 2000 677]/FontName/Times#20New#20Roman,Bold/FontWeight 700/ItalicAngle 0/Leadi<</BaseFont/Times#20New#20Roman/Encoding/WinAnsiEncoding/FirstChar 32/FontDescri<</Ascent 891/AvgWidth 401/CapHeight 693/Descent -216/Flags 32/FontBBox[-568 -216 2000 693]/FontName/Times#20New#20Roman/FontWeight 400/ItalicAngle 0/Leading 42<</BaseFont/Arial,Bold/Encoding/WinAnsiEncoding/FirstChar 32/FontDescriptor 10 0<</Ascent 905/AvgWidth 479/CapHeight 728/Descent -210/Flags 32/FontBBox[-628 -210 2000 728]/FontName/Arial,Bold/FontWeight 700/ItalicAngle 0/Leading 33/MaxWidth<</BaseFont/Times#20New#20Roman,Italic/Encoding/WinAnsiEncoding/FirstChar 32/FontDescriptor 12 0 R/LastChar 118/Name/F4/Subtype/TrueType/Type/Font/Widths 164 0 <</Ascent 891/AvgWidth 402/CapHeight 694/Descent -216/Flags 32/FontBBox[-498 -216 1333 694]/FontName/Times#20New#20Roman,Italic/FontWeight 400/ItalicAngle -16.4<</BaseFont/Arial/Encoding/WinAnsiEncoding/FirstChar 32/FontDescriptor 14 0 R/La<</Ascent 905/AvgWidth 441/CapHeight 728/Descent -210/Flags 32/FontBBox[-665 -210 2000 728]/FontName/Arial/FontWeight 400/ItalicAngle 0/Leading 33/MaxWidth 2665<</Contents 16 0 R/Group<</CS/DeviceRGB/S/Transparency/Type/Group>>/MediaBox[0 0 612 792]/Parent 2 0 R/Resources<</Font<</F1 5 0 R/F2 7 0 R/F5 13 0 R>>/ProcSet[<</Filter/FlateDecode/Length 7534>>streamarents 1/Tabs/S/Type/Page>>
<</Contents 18 0 R/Group<</CS/DeviceRGB/S/Transparency/Type/Group>>/MediaBox[0 0 612 792]/Parent 2 0 R/Resources<</Font<</F1 5 0 R/F2 7 0 R/F5 13 0 R>>/ProcSet[<</Filter/FlateDecode/Length 6137>>streamarents 2/Tabs/S/Type/Page>>
<</Contents 20 0 R/Group<</CS/DeviceRGB/S/Transparency/Type/Group>>/MediaBox[0 0 612 792]/Parent 2 0 R/Resources<</Font<</F1 5 0 R/F2 7 0 R/F5 13 0 R/F6 21 0 R><</Filter/FlateDecode/Length 6533>>stream>>/StructParents 3/Tabs/S/Type/Page>>
<</BaseFont/Times#20New#20Roman/DescendantFonts 22 0 R/Encoding/Identity-H/Subty<</BaseFont/Times#20New#20Roman/CIDSystemInfo 24 0 R/CIDToGIDMap/Identity/DW 100<</Ascent 891/AvgWidth 401/CapHeight 693/Descent -216/Flags 32/FontBBox[-568 -216 2000 693]/FontFile2 160 0 R/FontName/Times#20New#20Roman/FontWeight 400/Italic<</Contents 27 0 R/Group<</CS/DeviceRGB/S/Transparency/Type/Group>>/MediaBox[0 0 612 792]/Parent 2 0 R/Resources<</ExtGState<</GS28 28 0 R/GS29 29 0 R>>/Font<</F1 5 0 R/F2 7 0 R/F3 9 0 R/F5 13 0 R/F6 21 0 R>>/ProcSet[/PDF/Text/ImageB/ImageC<</Filter/FlateDecode/Length 5369>>streamge>>

The number of images per page are much bigger (about one per page)!

Shellscript

• If a pdf file contains an image (inserted in a document alongside text or as whole pages, 'scanned pdf'), the file often (maybe always) contains the string /Image/.




• In the same way you can search for the string /Text to tell if a pdf file contains text (not scanned).

I made the shellscript pdf-text-or-image, and it might work in most cases with your files. The shellscript looks for the text strings /Image/ and /Text in the pdf files.

#!/bin/bash

echo "shellscript $0"
ls --color --group-directories-first
read -p "Is it OK to use this shellscript in this directory? (y/N) " ans
if [ "$ans" != "y" ]
then
 exit
fi

mkdir -p scanned
mkdir -p text
mkdir -p "s-and-t"

for file in *.pdf
do
 grep -aq '/Image/' "$file"
 if [ $? -eq 0 ]
 then
 image=true
 else
 image=false
 fi
 grep -aq '/Text' "$file"
 if [ $? -eq 0 ]
 then
 text=true
 else
 text=false
 fi


 if $image && $text
 then
 mv "$file" "s-and-t"
 elif $image
 then
 mv "$file" "scanned"
 elif $text
 then
 mv "$file" "text"
 else
 echo "$file undecided"
 fi
done

Make the shellscript executable,

chmod ugo+x pdf-text-or-image

Change directory to where you have the pdf files and run the shellscript.

Identified files are moved to the following subdirectories

• scanned


• text


• 
  s-and-t (for documents with both [scanned?] images and text content)

Unidentified file objects, 'UFOs', remain in the current directory.

Test

I tested the shellscript with two of your files, AR-G1002.pdf and AR-G1003.pdf, and with some own pdf files (that I have created using Libre Office Impress).

$ ./pdf-text-or-image
shellscript ./pdf-text-or-image
s-and-t mkUSB-quick-start-manual-11.pdf mkUSB-quick-start-manual-nox-11.pdf
scanned mkUSB-quick-start-manual-12-0.pdf mkUSB-quick-start-manual-nox.pdf
text mkUSB-quick-start-manual-12.pdf mkUSB-quick-start-manual.pdf
AR-G1002.pdf mkUSB-quick-start-manual-74.pdf OBI-quick-start-manual.pdf
AR-G1003.pdf mkUSB-quick-start-manual-75.pdf oem.pdf
DescriptionoftheOneButtonInstaller.pdf mkUSB-quick-start-manual-8.pdf pdf-text-or-image
GrowIt.pdf mkUSB-quick-start-manual-9.pdf pdf-text-or-image0
list-files.pdf mkUSB-quick-start-manual-bas.pdf README.pdf
Is it OK to use this shellscript in this directory? (y/N) y

$ ls -1 *
pdf-text-or-image
pdf-text-or-image0

s-and-t:
DescriptionoftheOneButtonInstaller.pdf
GrowIt.pdf
mkUSB-quick-start-manual-11.pdf
mkUSB-quick-start-manual-12-0.pdf
mkUSB-quick-start-manual-12.pdf
mkUSB-quick-start-manual-8.pdf
mkUSB-quick-start-manual-9.pdf
mkUSB-quick-start-manual.pdf
OBI-quick-start-manual.pdf
README.pdf

scanned:
AR-G1002.pdf

text:
AR-G1003.pdf
list-files.pdf
mkUSB-quick-start-manual-74.pdf
mkUSB-quick-start-manual-75.pdf
mkUSB-quick-start-manual-bas.pdf
mkUSB-quick-start-manual-nox-11.pdf
mkUSB-quick-start-manual-nox.pdf
oem.pdf

Let us hope that

• there are no UFOs in your set of files


• the sorting is correct concerning text versus scanned/images

1. Put all the .pdf files in one folder.


2. No .txt file in that folder.


3. In terminal change directory to that folder with cd <path to dir>


4. Make one more directory for non scanned files. Example:

 mkdir ./x 
 for file in *.pdf; do
 if [ $(pdftotext "$file")"x" == "x" ] ; then mv "$file" ./x; fi
 rm *.txt
 done

All the pdf scanned files will remain in the folder and other files will move to another folder.

Hobbyist offers a good solution if the document collection's scanned documents do not have text added with optical character recognition (OCR). If this is a possibility, you may want to do some scripting that reads the output of pdfinfo -meta and checks for the tool used to create the file, or employ a Python routine that uses one of the Python libraries to examine them. Searching for text with a tool like strings will be unreliable because PDF content can be compressed. And checking the creation tool is not failsafe, either, since PDF pages can be combined; I routinely combine PDF text documents with scanned images to keep things together.

I'm sorry that I am unable to offer specific suggestions. It's been a while since I poked at the PDF internal structure, but depending on how stringent your requirements are, you may want to know that it's kind of complicated. Good luck!

If this is more about actually detecting if PDF was created by scanning rather than pdf has images instead of text then you might need to dig into the metadata of the file, not just content.

In general, for the files I could find on my computer and your test files, following is true:

• Scanned files have less than 1000chars/page vs. non scanned ones who always have more than 1000chars/page


• Multiple independent scanned files had "Canon" listed as the PDF creator, probably referencing Canon scanner software


• PDFs with "Microsoft Word" as creator are likely to not be scanned, as they are word exports. But someone could scan to word, then export to PDF - some people have very strange workflow.

I'm using Windows at the moment, so I used node.js for the following example:

const fs = require("mz/fs");
const pdf_parse = require("pdf-parse");
const path = require("path");


const SHOW_SCANNED_ONES = process.argv.indexOf("scanned") != -1;

const DEBUG = process.argv.indexOf("debug") != -1;
const STRICT = process.argv.indexOf("strict") != -1;

const debug = DEBUG ? console.error : () => ;

(async () => 
 const pdfs = (await fs.readdir(".")).filter((fname) => return fname.endsWith(".pdf") );

 for (let i = 0, l = pdfs.length; i < l; ++i) 
 const pdffilename = pdfs[i];
 try 
 debug("nnFILE: ", pdffilename);
 const buffer = await fs.readFile(pdffilename);
 const data = await pdf_parse(buffer);

 if (!data.info)
 data.indo = ;
 if (!data.metadata) 
 data.metadata = 
 _metadata: 
 ;
 


 // PDF info
 debug(data.info);
 // PDF metadata
 debug(data.metadata);
 // text length
 const textLen = data.text ? data.text.length : 0;
 const textPerPage = textLen / (data.numpages);
 debug("Text length: ", textLen);
 debug("Chars per page: ", textLen / data.numpages);
 // PDF.js version
 // check https://mozilla.github.io/pdf.js/getting_started/
 debug(data.version);

 if (evalScanned(data, textLen, textPerPage) == SHOW_SCANNED_ONES) 
 console.log(path.resolve(".", pdffilename));
 
 
 catch (e) 
 if (strict && !debug) 
 console.error("Failed to evaluate " + item);
 
 
 debug("Failed to evaluate " + item);
 debug(e.stack);
 
 if (strict) 
 process.exit(1);
 
 
 
)();
const IS_CREATOR_CANON = /canon/i;
const IS_CREATOR_MS_WORD = /microsoft.*?word/i;
// just defined for better clarity or return values
const IS_SCANNED = true;
const IS_NOT_SCANNED = false;
function evalScanned(pdfdata, textLen, textPerPage) 
 if (textPerPage < 300 && pdfdata.numpages>1) 
 // really low number, definitelly not text pdf
 return IS_SCANNED;
 
 // definitelly has enough text
 // might be scanned but OCRed
 // we return this if no 
 // suspition of scanning is found
 let implicitAssumption = textPerPage > 1000 ? IS_NOT_SCANNED : IS_SCANNED;
 if (IS_CREATOR_CANON.test(pdfdata.info.Creator)) 
 // this is always scanned, canon is brand name
 return IS_SCANNED;
 
 return implicitAssumption;