Untitled

package lia.analysis;

import java.io.File;
import java.io.FileReader;
import java.io.IOException;



import org.apache.lucene.analysis.SimpleAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Version;


public class SimpleFileIndexer {
	
	
	public static void main(String[] args) throws Exception   {
		
		File indexDir = new File("C:/Users/Raden/Documents/myindex2");
		File dataDir = new File("C:/xampp/htdocs/rsj/Document/cranDATA/cran");
		String suffix = "txt";
		
		SimpleFileIndexer indexer = new SimpleFileIndexer();
		
		int numIndex = indexer.index(indexDir, dataDir, suffix);
		
		System.out.println("Total files indexed " + numIndex);
		
	}

	/*
	private int index(File indexDir, File dataDir, String suffix) throws Exception {

	    IndexWriter index = new IndexWriter(FSDirectory.open(indexDir),
	        new StandardAnalyzer(Version.LUCENE_CURRENT), false,  
	        IndexWriter.MaxFieldLength.UNLIMITED);

	    // tweak the settings for your hardware
	    index.setUseCompoundFile(false);
	    index.setRAMBufferSizeMB(256);
	    index.setMergeFactor(30);

	    indexDirectory(index, dataDir, suffix);

	    index.optimize();
	    int numIndexed = index.maxDoc();
	    index.close();

	    // you'll need to update indexDirectory() to keep track of indexed files
	    return numIndexed;
	}*/
	
	private int index(File indexDir, File dataDir, String suffix) throws Exception {
	    RAMDirectory ramDir = new RAMDirectory();
	    IndexWriter indexWriter = new IndexWriter(ramDir,
	        new StandardAnalyzer(Version.LUCENE_CURRENT), true,  
	        IndexWriter.MaxFieldLength.UNLIMITED);
	    indexWriter.setUseCompoundFile(false);
	    indexDirectory(indexWriter, dataDir, suffix);
	    int numIndexed = indexWriter.maxDoc();
	    indexWriter.optimize();
	    indexWriter.close();


	    IndexWriter index = new IndexWriter(FSDirectory.open(indexDir),
	        new StandardAnalyzer(Version.LUCENE_CURRENT), false,  
	        IndexWriter.MaxFieldLength.UNLIMITED);
	    index.addIndexesNoOptimize(ramDir);
	    index.optimize();
	    index.close();

	    return numIndexed;
	}
	
	
	private void indexDirectory(IndexWriter indexWriter, File dataDir, String suffix) throws IOException {
		File[] files = dataDir.listFiles();
		for (int i = 0; i < files.length; i++) {
			File f = files[i];
			if (f.isDirectory()) {
				indexDirectory(indexWriter, f, suffix);
			}
			else {
				indexFileWithIndexWriter(indexWriter, f, suffix);
			}
		}
	}
	
	private void indexFileWithIndexWriter(IndexWriter indexWriter, File f, String suffix) throws IOException {
		if (f.isHidden() || f.isDirectory() || !f.canRead() || !f.exists()) {
			return;
		}
		if (suffix!=null && !f.getName().endsWith(suffix)) {
			return;
		}
		System.out.println("Indexing file " + f.getCanonicalPath());
		
		Document doc = new Document();
		doc.add(new Field("contents", new FileReader(f)));		
		doc.add(new Field("filename", f.getCanonicalPath(), Field.Store.YES, Field.Index.ANALYZED));
		indexWriter.addDocument(doc);
	} }