| 1 | package felix.task; |
| 2 | |
| 3 | import java.io.BufferedReader; |
| 4 | |
| 5 | import tuffy.util.DebugMan; |
| 6 | import tuffy.util.ExceptionMan; |
| 7 | import tuffy.util.FileMan; |
| 8 | import tuffy.util.UIMan; |
| 9 | import felix.dstruct.FelixQuery; |
| 10 | import felix.parser.FelixInputParser; |
| 11 | import felix.util.FelixConfig; |
| 12 | |
| 13 | /** |
| 14 | * Evidence loader that can run concurrently. |
| 15 | * It takes as input a filename and |
| 16 | * a FelixQuery which it uses to parse the evidence file. |
| 17 | * @author Ce Zhang |
| 18 | * |
| 19 | */ |
| 20 | public class ConcurrentEvidenceLoader extends Thread{ |
| 21 | |
| 22 | /** |
| 23 | * Path to the evidence file. |
| 24 | */ |
| 25 | String evidenceFileName = ""; |
| 26 | |
| 27 | /** |
| 28 | * FelixQuery |
| 29 | */ |
| 30 | FelixQuery fq = null; |
| 31 | |
| 32 | /** |
| 33 | * The constructor. |
| 34 | * @param _fileName |
| 35 | * @param _fq |
| 36 | */ |
| 37 | public ConcurrentEvidenceLoader(String _fileName, FelixQuery _fq){ |
| 38 | this.evidenceFileName = _fileName; |
| 39 | this.fq = _fq; |
| 40 | } |
| 41 | |
| 42 | /** |
| 43 | * Parses evidence file |
| 44 | */ |
| 45 | public void run(){ |
| 46 | |
| 47 | FelixInputParser parser = new FelixInputParser(this.fq); |
| 48 | |
| 49 | int chunkSize = FelixConfig.evidence_file_chunk_size;; |
| 50 | String f = this.evidenceFileName; |
| 51 | String g = FileMan.getGZIPVariant(f); |
| 52 | if (g == null) { |
| 53 | ExceptionMan.die("File does not exist: " + f); |
| 54 | } else { |
| 55 | f = g; |
| 56 | } |
| 57 | UIMan.println(">>> Parsing evidence file: " + f); |
| 58 | if (FileMan.getFileSize(f) <= chunkSize) { |
| 59 | parser.parseEvidenceFile(f); |
| 60 | } else { |
| 61 | try { |
| 62 | long lineOffset = 0, lastChunkLines = 0; |
| 63 | BufferedReader reader = FileMan.getBufferedReaderMaybeGZ(f); |
| 64 | StringBuilder sb = new StringBuilder(); |
| 65 | String line = reader.readLine(); |
| 66 | int ct = 0; |
| 67 | while (line != null) { |
| 68 | |
| 69 | ct ++; |
| 70 | if(ct % 100000 == 0){ |
| 71 | UIMan.verbose(3, "LINE_" + ct); |
| 72 | } |
| 73 | |
| 74 | sb.append(line); |
| 75 | sb.append("\n"); |
| 76 | lastChunkLines++; |
| 77 | if (sb.length() >= chunkSize) { |
| 78 | parser.parseEvidenceString(sb.toString(), |
| 79 | lineOffset); |
| 80 | sb.delete(0, sb.length()); |
| 81 | sb = new StringBuilder(); |
| 82 | lineOffset += lastChunkLines; |
| 83 | lastChunkLines = 0; |
| 84 | UIMan.print("."); |
| 85 | } |
| 86 | line = reader.readLine(); |
| 87 | } |
| 88 | reader.close(); |
| 89 | if (sb.length() > 0) { |
| 90 | parser.parseEvidenceString(sb.toString(), lineOffset); |
| 91 | } |
| 92 | UIMan.println(); |
| 93 | } catch (Exception e) { |
| 94 | ExceptionMan.handle(e); |
| 95 | } |
| 96 | } |
| 97 | try { |
| 98 | DebugMan.runGC(); |
| 99 | } catch (Exception e) { |
| 100 | e.printStackTrace(); |
| 101 | } |
| 102 | |
| 103 | } |
| 104 | |
| 105 | |
| 106 | } |