Class | XhochY::Drossellog::LogFileParser |
In: |
xhochy/drossellog/logfileparser.rb
|
Parent: | Object |
Parses a hole Apache combined log file
filename | [R] | The name of the logfile which is processed with a certain instance |
sha1sum | [R] | The SHA1 checksum of the logfile |
Remove the old, outdated data
# File xhochy/drossellog/logfileparser.rb, line 54 54: def cleanup_old 55: path = File.join(@tmp_path, File.basename(@filename)); 56: 57: # remove old directory 58: if File.exist?(path) 59: FileUtils.rm_r(path) 60: puts path 61: end 62: 63: # make new 64: FileUtils.mkdir_p(path) 65: end
Generate the SHA1 checksum of the log file
# File xhochy/drossellog/logfileparser.rb, line 25 25: def generate_sha1sum 26: s = File.read(@filename) 27: @sha1sum = Digest::SHA1.hexdigest(s) 28: end
Checks if a directory already exists and if the content is outdated
# File xhochy/drossellog/logfileparser.rb, line 31 31: def outdated? 32: if @sha1sum.empty? 33: generate_sha1sum 34: end 35: 36: path = File.join(@tmp_path, File.basename(@filename)); 37: path2 = File.join(path, 'sha1sum') 38: 39: if File.exist?(path) # look for tmp-directory 40: 41: if File.exist?(path2) # look for sha1sum file 42: sum = File.read(path2).strip 43: return !sum.eql?(@sha1sum) 44: else # sha1sum file does not exist 45: return true 46: end 47: 48: else # tmp-directory does not exist 49: return true 50: end 51: end
Parse the complete file and store the Lines in an array. If a line could not be parsed(most times it is not in the correct format) it will be outputted to stdout. If you think the line was well formed check the regular expression in xhochy/drossellog/logline.rb and try to correct it, if it is really something missing, please report this as a bug.
# File xhochy/drossellog/logfileparser.rb, line 73 73: def parse 74: path = File.join(@tmp_path, File.basename(@filename)); 75: path2 = File.join(path, 'sha1sum') 76: 77: File.open(@filename, 'r') do |f| 78: @lines = [] 79: 80: while line = f.gets 81: if line =~ XhochY::Drossellog::LogLine.get_rxp 82: line_obj = XhochY::Drossellog::LogLine.new(line) 83: @lines << line_obj 84: else # parse line 85: puts line 86: end 87: end 88: end 89: end
Save the data into sperate YAML-Files
The filenames consist of the following: <tmp_path>/<year>/<month>/<day>.yml
# File xhochy/drossellog/logfileparser.rb, line 94 94: def save_data(domain, domain_alias) 95: sorted = {} 96: @lines.each do |line| 97: if sorted[line.year] == nil 98: sorted[line.year] = {} 99: end 100: 101: if sorted[line.year][line.month] == nil 102: sorted[line.year][line.month] = {} 103: end 104: 105: if sorted[line.year][line.month][line.day] == nil 106: sorted[line.year][line.month][line.day] = [] 107: end 108: 109: sorted[line.year][line.month][line.day] << line 110: end #^ @lines.each 111: 112: sorted.each do |year, value| 113: value.each do |month, value2| 114: value2.each do |day, value3| 115: path = File.join(@tmp_path, File.basename(@filename), year.to_s, month) 116: filename = File.join(path, day.to_s) + '.yml' 117: FileUtils.mkdir_p(path) 118: 119: outdoc = { 120: 'day' => day.to_i, 121: 'month' => month, 122: 'year' => year.to_i, 123: 'domain' => domain, 124: 'alias' => domain_alias, 125: 'lines' => [] 126: } 127: 128: strip_tags = ['day', 'month', 'year'] 129: value3.each do |line| 130: hash = line.to_hash strip_tags 131: outdoc['lines'] << hash 132: end 133: 134: File.open(filename, 'w') do |f| 135: f.write outdoc.to_yaml 136: end 137: 138: end #^ value2.each 139: end #^ value.each 140: end #^ sorted.each 141: 142: path = File.join(@tmp_path, File.basename(@filename), 'sha1sum') 143: File.open(path, 'w') do |f| 144: f.write(@sha1sum) 145: end 146: end