Notes on Tika

package com.etretatlogiciels.tika;

import java.io.File;
import java.io.IOException;

import org.junit.Test;

import static org.junit.Assert.assertTrue;

import org.apache.tika.Tika;

/**
 * @author Russell Bateman
 * @since September 2016
 */
public class MimeDetectionTest
{
  @Test
  public void testByOpenFile() throws IOException
  {
    File file = new File( "src/test/resources/Statement_201412.pdf" );
    Tika tika = new Tika();

    String filetype = tika.detect( file );
    System.out.println( "File type is " + filetype + "." );

    assertTrue( "Wrong MIME type", filetype.equals( "application/pdf" ) );
  }

  @Test
  public void testByData()
  {
    final String text = "This is a test.";

    Tika tika = new Tika();

    String filetype = tika.detect( text.getBytes() );
    System.out.println( "Content type is " + filetype + "." );

    assertTrue( "Wrong MIME type", filetype.equals( "text/plain" ) );
  }
}

package com.etretatlogiciels.tika;

import org.junit.Test;

import static org.junit.Assert.assertTrue;

import org.apache.tika.language.LanguageIdentifier;

/**
 * Guess what? This simply doesn't work--what did I miss here?
 * @author Russell Bateman
 * @since September 2016
 */
@SuppressWarnings( "deprecation" )
public class LanguageDetectionTest
{
  @Test
  public void testEnglish()
  {
    LanguageIdentifier identifier = new LanguageIdentifier( "this is english" );
    String language = identifier.getLanguage();
    System.out.println( "Language of string content is " + language );

    assertTrue( "Wrong language", language.equals( "en" ) );
  }

  /**
   * Well, this certainly doesn't work.
   */
  @Test
  public void testFrench()
  {
    LanguageIdentifier identifier = new LanguageIdentifier( "ce texte est en français" );
    String language = identifier.getLanguage();
    System.out.println( "Language of string content is " + language );

    assertTrue( "Wrong language", language.equals( "fr" ) );
  }
}

package com.etretatlogiciels.tika;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;

import org.junit.Test;

import static org.junit.Assert.assertTrue;

import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.pdf.PDFParser;
import org.apache.tika.sax.BodyContentHandler;
import org.xml.sax.SAXException;

/**
 * @author Russell Bateman
 * @since September 2016
 */
public class ExtractPdfTest
{
  @Test
  public void test() throws IOException, TikaException, SAXException
  {
    BodyContentHandler handler  = new BodyContentHandler();
    Metadata           metadata = new Metadata();
    FileInputStream    stream   = new FileInputStream( new File( "src/test/resources/Statement_201412.pdf" ) );
    ParseContext       context  = new ParseContext();

    PDFParser parser = new PDFParser();
    parser.parse( stream, handler, metadata, context );

    String contents = handler.toString();

    System.out.println( "PDF contents:" );
    System.out.println( contents );
    System.out.println( "Metadata: " );

    String[]       metadataNames = metadata.names();
    List< String > names         = Arrays.asList( metadataNames );

    for( String name : names )
      System.out.println( "  " + name + " : " + metadata.get( name ) );

    assertTrue( "Not the expected content", contents.contains( "Bateman, Russell" ) );
    assertTrue( "Missing Content-Type",     names.contains( "Content-Type" ) );
    assertTrue( "Content-Type is not PDF",  metadata.get( "Content-Type" ).equals( "application/pdf" ) );
  }
}

package com.etretatlogiciels.tika;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import org.junit.BeforeClass;
import org.junit.Test;

import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.Parser;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.exception.TikaException;
import org.apache.tika.parser.ParseContext;
import org.xml.sax.SAXException;

/**
 * I undertook this to determine whether AutoDetectParser is preferable
 * over Tika Facade. On the face of it, it is. It does more types, one
 * need not retro-fit new types later, it's as fast, etc.
 * @author Russell Bateman
 * @since October 2016
 */
public class AutoDetectParserTest
{
  private static BodyContentHandler handler;
  private static Metadata           metadata;
  private static ParseContext       context;
  private static Parser             parser;

  private static final boolean VERBOSE = true;

  @BeforeClass
  public static void erectTikaParsingArtifacts()
  {
    long start = System.currentTimeMillis();
    // here we do not tell Tika we want to parse PDF; it will figure that out...
    // this statement takes about 3 seconds to execute:
    parser = new AutoDetectParser();
    System.out.println( "new AutoDetectParser()    elapsed time: " + ( System.currentTimeMillis() - start )
                      + " milliseconds (single instance saved for all tests)" );

    handler  = new BodyContentHandler();
    metadata = new Metadata();
    context  = new ParseContext();
  }

  private void printSummary()
  {
    String[]       metadataNames = metadata.names();
    List< String > names         = Arrays.asList( metadataNames );

    Collections.sort( names );

    if( VERBOSE )
    {
      System.out.println( metadata.get( "Content-Type" ) + " contents:" );
      System.out.println( handler.toString() );
      System.out.println( "Metadata: " );

      for( String name : names )
        System.out.println( "  " + name + " : " + metadata.get( name ) );
    }
  }

  @Test
  public void testPdf() throws IOException, TikaException, SAXException
  {
    FileInputStream stream = new FileInputStream( new File( "src/test/resources/Statement_201412.pdf" ) );

    // this statement takes roughly 7 seconds to execute:
    long start = System.currentTimeMillis();
    parser.parse( stream, handler, metadata, context );
    System.out.println( "parser.parse( small PDF ) elapsed time: "
                      + ( System.currentTimeMillis() - start ) + " milliseconds" );

    printSummary();
  }

  @Test
  public void testXml() throws IOException, TikaException, SAXException
  {
    FileInputStream stream = new FileInputStream( new File( "pom.xml" ) );

    long start = System.currentTimeMillis();
    parser.parse( stream, handler, metadata, context );
    System.out.println( "parser.parse( pom.xml )   elapsed time:  "
                      + ( System.currentTimeMillis() - start ) + " milliseconds" );
    printSummary();
  }

  @Test
  public void testHtml() throws IOException, TikaException, SAXException
  {
    FileInputStream stream = new FileInputStream( new File( "src/test/resources/sample.html" ) );

    long start = System.currentTimeMillis();
    parser.parse( stream, handler, metadata, context );
    System.out.println( "parser.parse( tiny HTML ) elapsed time: "
                      + ( System.currentTimeMillis() - start ) + " milliseconds" );
    printSummary();
  }

  @Test
  public void testText() throws IOException, TikaException, SAXException
  {
    FileInputStream stream = new FileInputStream( new File( "src/test/resources/sample.txt" ) );

    long start = System.currentTimeMillis();
    parser.parse( stream, handler, metadata, context );
    System.out.println( "parser.parse( tiny text ) elapsed time:   "
                      + ( System.currentTimeMillis() - start ) + " milliseconds" );
    printSummary();
  }

  @Test
  public void testJpeg() throws IOException, TikaException, SAXException
  {
    FileInputStream stream = new FileInputStream( new File( "src/test/resources/test-first-code-second.jpg" ) );

    long start = System.currentTimeMillis();
    parser.parse( stream, handler, metadata, context );
    System.out.println( "parser.parse( JPeG )      elapsed time: "
                      + ( System.currentTimeMillis() - start ) + " milliseconds" );
    printSummary();
  }

  @Test
  public void testPng() throws IOException, TikaException, SAXException
  {
    FileInputStream stream = new FileInputStream( new File( "src/test/resources/etl-fhir-quiver.png" ) );

    long start = System.currentTimeMillis();
    parser.parse( stream, handler, metadata, context );
    System.out.println( "parser.parse( PNG )       elapsed time: "
                      + ( System.currentTimeMillis() - start ) + " milliseconds" );
    printSummary();
  }
}