数码吧

Tika文本提取工具的使用(word、pdf、excel等)

Tika是Apache的Lucene项目下面的子项目，在lucene的应用中可以使用tika获取大批量文档中的内容来建立索引，非常方便，也很容易使用~

Tika的缺点就是都是依赖外部的jar包，导致jar包的重量太大，lucene的核心包只有1M，tika约20M，tika依赖的外部的jar包有多样的功能，比如PDFBox和Apache POI能获取文档的字体，布置和内置图片信息，而Tika只是获取文本信息。但是这些外部的jar包又没有把获取文本信息的抽离出一个单独的jar包。

1、Tika的作用

工程结构：

2、Tika的工具类

1. package org.lucene.util;
2. import java.io.File;
3. import java.io.FileInputStream;
4. import java.io.FileNotFoundException;
5. import java.io.IOException;
6. import java.io.InputStream;
7. import org.apache.lucene.document.Document;
8. import org.apache.lucene.document.Field;
9. import org.apache.lucene.index.CorruptIndexException;
10. import org.apache.lucene.index.IndexWriter;
11. import org.apache.lucene.index.IndexWriterConfig;
12. import org.apache.lucene.store.Directory;
13. import org.apache.lucene.store.FSDirectory;
14. import org.apache.lucene.store.LockObtainFailedException;
15. import org.apache.lucene.util.Version;
16. import org.apache.tika.Tika;
17. import org.apache.tika.exception.TikaException;
18. import org.apache.tika.metadata.Metadata;
19. import org.apache.tika.parser.AutoDetectParser;
20. import org.apache.tika.parser.ParseContext;
21. import org.apache.tika.parser.Parser;
22. import org.apache.tika.sax.BodyContentHandler;
23. import org.xml.sax.ContentHandler;
24. import org.xml.sax.SAXException;
25. import com.chenlb.mmseg4j.analysis.MMSegAnalyzer;
27. public class IndexUtil {
28. /**
29. * 直接读取pdf建立索引，结果是索引建立成功了，但是索引存储的数据却是乱的
30. */
31. public void index() {
32. try {
33. File f = new File("F:文档资料lucene_in_action中文版.pdf");
34. Directory dir = FSDirectory.open(new File("f:/lucene"));
35. IndexWriter writer = new IndexWriter(dir,new IndexWriterConfig(Version.LUCENE_35, new MMSegAnalyzer()));
36. writer.deleteAll();
37. Document doc = new Document();
38. doc.add(new Field("content",new Tika().parse(f)));
39. writer.addDocument(doc);
40. writer.close();
41. } catch (CorruptIndexException e) {
42. e.printStackTrace();
43. } catch (LockObtainFailedException e) {
44. e.printStackTrace();
45. } catch (FileNotFoundException e) {
46. e.printStackTrace();
47. } catch (IOException e) {
48. e.printStackTrace();
49. }
50. }
52. /**
53. * 根据Tika得到文档的内容，这种比下面那种获取的要简单很多，
54. * 据tika的文档上说，效率没有下面的那种高，可能封装的比较多
55. * @param f
56. * @return
57. * @throws IOException
58. * @throws TikaException
59. */
60. public String tikaTool(File f) throws IOException, TikaException {
61. Tika tika = new Tika();
62. Metadata metadata = new Metadata();
63. metadata.set(Metadata.AUTHOR, "空号");//重新设置文档的媒体内容
64. metadata.set(Metadata.RESOURCE_NAME_KEY, f.getName());
65. String str = tika.parseToString(new FileInputStream(f),metadata);
66. for(String name:metadata.names()) {
67. System.out.println(name+":"+metadata.get(name));
68. }
69. return str;
70. }
72. /**
73. * 根据Parser得到文档的内容
74. * @param f
75. * @return
76. */
77. public String fileToTxt(File f) {
78. Parser parser = new AutoDetectParser();//自动检测文档类型，自动创建相应的解析器
79. InputStream is = null;
80. try {
81. Metadata metadata = new Metadata();
82. metadata.set(Metadata.AUTHOR, "空号");//重新设置文档的媒体内容
83. metadata.set(Metadata.RESOURCE_NAME_KEY, f.getName());
84. is = new FileInputStream(f);
85. ContentHandler handler = new BodyContentHandler();
86. ParseContext context = new ParseContext();
87. context.set(Parser.class,parser);
88. parser.parse(is,handler, metadata,context);
89. for(String name:metadata.names()) {
90. System.out.println(name+":"+metadata.get(name));
91. }
92. return handler.toString();
93. } catch (FileNotFoundException e) {
94. e.printStackTrace();
95. } catch (IOException e) {
96. e.printStackTrace();
97. } catch (SAXException e) {
98. e.printStackTrace();
99. } catch (TikaException e) {
100. e.printStackTrace();
101. } finally {
102. try {
103. if(is!=null) is.close();
104. } catch (IOException e) {
105. e.printStackTrace();
106. }
107. }
108. return null;
109. }
110. }

package org.lucene.util;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.Version;
import org.apache.tika.Tika;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.sax.BodyContentHandler;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import com.chenlb.mmseg4j.analysis.MMSegAnalyzer;

public class IndexUtil {
	/**
	 * 直接读取pdf建立索引，结果是索引建立成功了，但是索引存储的数据却是乱的
	 */
	public void index() {
		try {
			File f = new File("F:文档资料lucene_in_action中文版.pdf");
			Directory dir = FSDirectory.open(new File("f:/lucene"));
			IndexWriter writer = new IndexWriter(dir,new IndexWriterConfig(Version.LUCENE_35, new MMSegAnalyzer()));
			writer.deleteAll();
			Document doc = new Document();
			doc.add(new Field("content",new Tika().parse(f)));
			writer.addDocument(doc);
			writer.close();
		} catch (CorruptIndexException e) {
			e.printStackTrace();
		} catch (LockObtainFailedException e) {
			e.printStackTrace();
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	/**
	 * 根据Tika得到文档的内容，这种比下面那种获取的要简单很多，
	 * 据tika的文档上说，效率没有下面的那种高，可能封装的比较多
	 * @param f
	 * @return
	 * @throws IOException
	 * @throws TikaException
	 */
	public String tikaTool(File f) throws IOException, TikaException {
		Tika tika = new Tika();
		Metadata metadata = new Metadata();
		metadata.set(Metadata.AUTHOR, "空号");//重新设置文档的媒体内容
		metadata.set(Metadata.RESOURCE_NAME_KEY, f.getName());
		String str = tika.parseToString(new FileInputStream(f),metadata);
		for(String name:metadata.names()) {
			System.out.println(name+":"+metadata.get(name));
		}
		return str;
	}
	
	/**
	 * 根据Parser得到文档的内容
	 * @param f
	 * @return
	 */
	public String fileToTxt(File f) {
		Parser parser = new AutoDetectParser();//自动检测文档类型，自动创建相应的解析器
		InputStream is = null;
		try {
			Metadata metadata = new Metadata();
			metadata.set(Metadata.AUTHOR, "空号");//重新设置文档的媒体内容
			metadata.set(Metadata.RESOURCE_NAME_KEY, f.getName());
			is = new FileInputStream(f);
			ContentHandler handler = new BodyContentHandler();
			ParseContext context = new ParseContext();
			context.set(Parser.class,parser);
			parser.parse(is,handler, metadata,context);
			for(String name:metadata.names()) {
				System.out.println(name+":"+metadata.get(name));
			}
			return handler.toString();
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} catch (SAXException e) {
			e.printStackTrace();
		} catch (TikaException e) {
			e.printStackTrace();
		} finally {
			try {
				if(is!=null) is.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		return null;
	}
}

1. package org.lucene.test;
3. import java.io.File;
4. import java.io.IOException;
5. import org.apache.tika.exception.TikaException;
6. import org.junit.Test;
7. import org.lucene.util.IndexUtil;
9. public class TestIndex {
10. @Test
11. public void testIndex() {
12. IndexUtil iu = new IndexUtil();
13. iu.index();
14. }
16. @Test
17. public void testTika01() {
18. IndexUtil iu = new IndexUtil();
19. System.out.println(iu.fileToTxt(new File("F:文档资料lucene_in_action中文版.pdf")));
20. }
22. @Test
23. public void testToka02() throws IOException, TikaException {
24. IndexUtil iu = new IndexUtil();
25. System.out.println(iu.tikaTool(new File("F:文档资料初级SQL开发指南.doc")));
26. }
27. }

package org.lucene.test;

import java.io.File;
import java.io.IOException;
import org.apache.tika.exception.TikaException;
import org.junit.Test;
import org.lucene.util.IndexUtil;

public class TestIndex {
	@Test
	public void testIndex() {
		IndexUtil iu = new IndexUtil();
		iu.index();
	}
	
	@Test
	public void testTika01() {
		IndexUtil iu = new IndexUtil();
		System.out.println(iu.fileToTxt(new File("F:文档资料lucene_in_action中文版.pdf")));
	}
	
	@Test
	public void testToka02() throws IOException, TikaException {
		IndexUtil iu = new IndexUtil();
		System.out.println(iu.tikaTool(new File("F:文档资料初级SQL开发指南.doc")));
	}
}

工程路径：http://download.csdn.net/detail/wxwzy738/5328383