last changes

pom.xml

@@ -24,6 +24,12 @@
   	<artifactId>gs-core</artifactId>
   	<version>1.2</version>
   </dependency>
+  <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
+ <dependency>
+    <groupId>org.apache.commons</groupId>
+    <artifactId>commons-lang3</artifactId>
+    <version>3.4</version>
+  </dependency>
   <dependency>
   	<groupId>org.jgrapht</groupId>
   	<artifactId>jgrapht-core</artifactId>

src/graph/extractor/graph/builder/BuilderFactory.java

@@ -1,17 +1,11 @@
 package graph.extractor.graph.builder;
 
 import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Random;
-
-import org.apache.lucene.index.IndexWriter;
-
-import dao.connection.LuceneConnector;
+import java.util.Random;	
 import dao.entity.DOProcess;
-import dao.lucene.tables.DAOJudge;
+import dao.lucene.cache.Cache;
 import graph.input.file.InputFactory;
 import graph.output.OutPutFactory;
 
@@ -19,7 +13,9 @@ public abstract class BuilderFactory {
 
 	protected List<DOProcess> processes;
 	protected InputFactory input;
-
+	public static long processIndex = 1;
+	
+	
 	public BuilderFactory(){
 		this.processes = new ArrayList<DOProcess>();
 
@@ -37,81 +33,125 @@ public abstract class BuilderFactory {
 		long originalSize = allDateFiles.size();
 		
 		
+
 		while(allDateFiles.size() > 0)
 		{
 			try{
 				Random rand = new Random();
-				
+
 				int randNumber = Math.abs(rand.nextInt(allDateFiles.size()));
 				File oneDayOfDO = allDateFiles.get(randNumber);
-				
+
 				List<String> allProcess  = input.getAllProcess(oneDayOfDO);
-				
+
 				System.out.println("Processando dia "+oneDayOfDO.getName() + "---> " + allProcess.size() +" processos");
 				System.out.println("Processados " +(originalSize - allDateFiles.size())+" de " + originalSize+" diarios");
+
+
+
 				for(String oneProcess: allProcess)
 				{
 					DOProcess process = this.extractProcess(oneProcess);
 					if(process != null)
 					{
 						process.setProcessDate(oneDayOfDO.getName());
+						process.setName(process.getProcessNumber() + oneDayOfDO.getName());
 						this.processes.add(process);
 					}
-					
-					
+
 				}
+
+
 				output.buildOutput(this.processes);
-				
+
+
+
 				//this.moveDoneFiles(oneDayOfDO);
 				this.processes.clear();
 				allProcess.clear();
 				allDateFiles.remove(oneDayOfDO);
 				
+				this.print();
+				this.store();
+				this.clear();
+				this.restart();
+				System.gc();
+
 			}catch(Exception e)
 			{
 				e.printStackTrace();
 			}
-			
+
 			System.out.println( "Completado " +(originalSize - allDateFiles.size())+" de " + originalSize +". Faltam "+ allDateFiles.size() +" Processos");
+	
+
+
+
 			System.out.println("\n\n\n\n");
-			
 
 		}
-		DAOJudge daoJudge = DAOJudge.getInstance();
-		daoJudge.store();
-		try {
-			LuceneConnector.getIndexDisk().commit();
-		} catch (IOException e) {
-			// TODO Auto-generated catch block
-			e.printStackTrace();
-		}
-		
-		
 
+		//		afterInsertAll();
 
-	}
+	
 
-	private void moveDoneFiles(File oneDayOfDO) {
-		
-		try{
-			
+
+
+
+
+
+
+	}
+	
+	private void restart() {
+		Cache.getCache();
 		
-		List<File> files = input.getFilesFromDate(oneDayOfDO);
+	}
+
+	private void clear() {
+		// TODO Auto-generated method stub
+		Cache.delete();
 		
 		
 		
-		for(File file: files)
-		{
-			File doneFolder = new File("/home/ailton/Monografia/done/"+file.getName());
-			Files.move(file.toPath(), doneFolder.toPath());
-			
-		}}catch(Exception e)
-		{
-			e.printStackTrace();
-		}
+	}
+
+	public void print()
+	{
+		Cache.getCache().printCache();
+	}
+	
+	
+	public void store()
+	{
+		Cache cache = Cache.getCache();
+		cache.store();
 		
 		
 	}
+	
+
+	//	private void moveDoneFiles(File oneDayOfDO) {
+	//		
+	//		try{
+	//			
+	//		
+	//		List<File> files = input.getFilesFromDate(oneDayOfDO);
+	//		
+	//		
+	//		
+	//		for(File file: files)
+	//		{
+	//			File doneFolder = new File("/home/ailton/Monografia/done/"+file.getName());
+	//			Files.move(file.toPath(), doneFolder.toPath());
+	//			
+	//		}}catch(Exception e)
+	//		{
+	//			e.printStackTrace();
+	//		}
+	//		
+	//		
+	//	}
 
 	public List<DOProcess> getAllProcess()
 	{

src/graph/extractor/graph/builder/STJBuilder.java

@@ -2,6 +2,8 @@ package graph.extractor.graph.builder;
 
 
 
+import org.apache.commons.lang3.StringUtils;
+
 import dao.entity.DOProcess;
 import dao.entity.Entity;
 import dao.entity.Judge;
@@ -16,6 +18,9 @@ public class STJBuilder extends BuilderFactory{
 	private String normalizeJudgeName(String judgeName)
 	{
 		
+	    judgeName = StringUtils.stripAccents(judgeName);
+
+		
 		if(judgeName.contains("(") && !judgeName.startsWith("(") )
 		{
 
@@ -44,7 +49,8 @@ public class STJBuilder extends BuilderFactory{
 	
 	private String normalizeEntityName(String entityName)
 	{
-		String originalEntityName = entityName.trim();
+		
+		entityName = StringUtils.stripAccents(entityName);
 		
 		if(entityName.contains("(PRESO)"))
 		{
@@ -62,6 +68,25 @@ public class STJBuilder extends BuilderFactory{
 		{
 			entityName = entityName.split("\\(")[0].trim();
 		}
+		if(entityName.contains("ASSIST"))
+		{
+			
+			entityName = entityName.split("ASSIST")[0].trim();
+			entityName = entityName.substring(0, entityName.length() -1);
+			
+			
+			
+		}
+		if(entityName.contains("CONVOCADA"))
+		{
+			
+			entityName = entityName.split("CONVOCADA")[0].trim();
+			entityName = entityName.substring(0, entityName.length() -1);
+			
+			
+			
+		}
+		
 		
 		
 		if(entityName.length() > 0)
@@ -75,9 +100,14 @@ public class STJBuilder extends BuilderFactory{
 	private Lawyer normalizeLawyerName(Lawyer lawyer)
 	{
 		
-		//String originalEntityName = entityName.trim();
+		
+		
 		String nameNormalized = lawyer.getName().replaceAll("\\r|\\n", " ").trim();
-
+		nameNormalized = StringUtils.stripAccents(nameNormalized);
+		
+		nameNormalized = nameNormalized.replaceAll("\"", "").trim();
+		nameNormalized = nameNormalized.replaceAll("/", "").trim();
+		 
 
 		try{
 			
@@ -92,6 +122,13 @@ public class STJBuilder extends BuilderFactory{
 				}
 				nameNormalized = nameNormalized.split("-")[0].trim();
 			}
+			if(nameNormalized.contains("ASSIST"))
+			{
+				
+				nameNormalized = nameNormalized.split("ASSIST")[0].trim();
+				nameNormalized = nameNormalized.substring(0, nameNormalized.length() -1);
+				
+			}
 			
 			if(nameNormalized.contains("E OUTRO"))
 			{
@@ -103,6 +140,36 @@ public class STJBuilder extends BuilderFactory{
 				nameNormalized = TYPES.NAMEDEFAULT;
 			}
 			
+			if(nameNormalized.contains("EM CAUSA")  )
+			{
+				nameNormalized = nameNormalized.split("EM CAUSA")[0].trim();
+				nameNormalized = nameNormalized.substring(0, nameNormalized.length() -1).trim();
+				
+			}
+				
+			
+			if(nameNormalized.endsWith(" E"))
+			{
+				nameNormalized = nameNormalized.split(" E")[0].trim();
+			}
+			
+			
+			if(this.getLowerCasePCT(nameNormalized) > 70)
+			{
+				nameNormalized = TYPES.NAMEDEFAULT;
+			}
+			
+			
+			if(nameNormalized.equals(""))
+			{
+				lawyer.setName(lawyer.getName());
+			}
+			else
+			{
+				lawyer.setName(nameNormalized);
+			}
+			
+			
 			
 		}
 		catch(Exception e)
@@ -112,37 +179,6 @@ public class STJBuilder extends BuilderFactory{
 		}
 
 
-
-		
-		
-		
-		if(nameNormalized.contains("EM CAUSA")  )
-		{
-			nameNormalized = nameNormalized.split("EM CAUSA")[0].trim();
-			nameNormalized = nameNormalized.substring(0, nameNormalized.length() -1).trim();
-			
-		}
-			
-		if(nameNormalized.equals("E OUTRO(S)"))
-		{
-			nameNormalized = TYPES.NAMEDEFAULT;
-		}
-			
-		
-		if(nameNormalized.endsWith(" E"))
-		{
-			nameNormalized = nameNormalized.split(" E")[0].trim();
-		}
-		
-		if(nameNormalized.equals(""))
-		{
-			lawyer.setName(lawyer.getName());
-		}
-		else
-		{
-			lawyer.setName(nameNormalized);
-		}
-		
 		return lawyer;
 		
 		
@@ -179,8 +215,12 @@ public class STJBuilder extends BuilderFactory{
 				if(type.contains("ADVOGAD") || type.contains("PROCURADOR") )
 				{
 				
-					
-					entity = this.normalizeLawyerName(new Lawyer(canditate.split(":")[1].trim()) );
+					Lawyer lawyer = this.normalizeLawyerName(new Lawyer(canditate.split(":")[1].trim()));
+					entity.setName( lawyer.getName());
+					if(lawyer.getOABNumber()!= null && lawyer.getOABNumber().equals(""))
+					{
+						entity.setOABNumber(lawyer.getOABNumber());
+					}
 					entity.setType(TYPES.LAWYER);
 					if(type.contains("ADVOGADOS") )
 						nextLawyer = true;
@@ -196,7 +236,8 @@ public class STJBuilder extends BuilderFactory{
 				{
 					nextLawyer = false;
 					entity.setType(TYPES.LAWYER);
-					entity.setName(canditate);
+					Lawyer lawyer = this.normalizeLawyerName(new Lawyer(canditate));
+					entity.setName(lawyer.getName());
 
 				}
 				else{
@@ -228,19 +269,6 @@ public class STJBuilder extends BuilderFactory{
 
 		DOProcess proc = new DOProcess();
 
-//		try{
-//			File file = new File("/home/ailton/Monografia/log");
-//			FileWriter fileWriter = new FileWriter(file, true);
-//			PrintWriter out = new PrintWriter(fileWriter);
-//			out.println(process);
-//			out.println();
-//			out.close();
-//
-//		}catch(Exception e)
-//		{
-//			e.printStackTrace();
-//		}
-
 
 		proc.setOrign("DJ_STJ");
 
@@ -273,6 +301,10 @@ public class STJBuilder extends BuilderFactory{
 
 			String processType = firstLine.toLowerCase().split("nº")[0].toLowerCase();
 			firstLine = firstLine.substring(processType.length());
+			processType = StringUtils.stripAccents(processType);
+			processType = processType.replaceAll("[^A-Za-z0-9 ]", "");
+
+			
 			ProcessTypes procType = new ProcessTypes(processType);
 			proc.setProcessType(procType);
 

src/graph/output/database/DatabaseOutput.java

@@ -1,15 +1,11 @@
 package graph.output.database;
 
-import java.io.IOException;
 import java.util.List;
 import java.util.Random;
 
-import dao.connection.LuceneConnector;
 import dao.entity.DOProcess;
 import dao.lucene.relations.DAOGeneralProcess;
-import dao.lucene.tables.DAOEntity;
-import dao.lucene.tables.DAOJudge;
-import dao.lucene.tables.DAOLawyer;
+
 import graph.output.OutPutFactory;
 
 public class DatabaseOutput extends OutPutFactory {
@@ -43,9 +39,9 @@ public class DatabaseOutput extends OutPutFactory {
 		
 		int counter  = 0;
 		this.runned = 0;
-		DAOJudge daoJudge = DAOJudge.getInstance();
-		DAOEntity daoEntity = DAOEntity.getInstance();
-		DAOLawyer daoLawyer = DAOLawyer.getInstance();
+//		DAOJudge daoJudge = DAOJudge.getInstance();
+//		DAOEntity daoEntity = DAOEntity.getInstance();
+//		DAOLawyer daoLawyer = DAOLawyer.getInstance();
 		
 		while(allProcess.size() > 0)
 		{
@@ -89,8 +85,18 @@ public class DatabaseOutput extends OutPutFactory {
 		
 		
 		System.out.println("Foram atualizados " +runned+". Skip: " + (processSize-runned));
-
-		System.gc();
+		
+		try{
+			
+			
+			
+			//LuceneConnector.getIndexDisk().commit();
+			System.gc();
+		}catch(Exception e)
+		{
+			e.printStackTrace();
+		}
+		
 		
 		
 		

test

@@ -0,0 +1,48 @@
+18102013
+28032016
+11022015
+20042015
+08042015
+04112014
+11072014
+29092015
+01092014
+04102013
+22042016
+13012014
+05112013
+10102013
+18122014
+13102015
+27122013
+04042016
+14102015
+15092015
+05122013
+09052016
+17032016
+03122014
+29092015
+25042014
+24102013
+04112015
+01102013
+14102015
+26052015
+25032015
+10072014
+29072014
+21012016
+06042016
+29112013
+22012016
+25072014
+10032014
+23102015
+01122014
+07102013
+01062015
+17032014
+28012014
+23082016
+28112014