stackdriver log converts to goaccess format

_stackdriver-log-to-goaccess.md

1. Use gcloud logging read to pull logs
2. Convert it to csv style via json2csv.rb

ruby json2csv.rb

3. Use goaccess to generate html output

goaccess -p goaccess.format -f output.csv -o output.html --hour-spec=min

goaccess.format

log-format %dT%t+%^,%h,%v,%m,%U,%L,%s
date-format %Y-%m-%d
time-format %T

json2csv.rb

require 'json'
require 'date'
require 'uri'

class JsonFileReader

  def initialize(file_path, &obj_callback)
    @file         = File.open(file_path)
    @obj_callback = obj_callback
  end

  def run
    brace = 0
    buffer = []
    read_size = 0
    total_size = @file.size
    prev_progress = 0

    while true
      c = @file.readchar
      read_size += 1

      progress = ((read_size * 100.0)/ total_size).to_i
      if prev_progress != progress
        prev_progress = progress
        puts "progress: #{progress}%"
      end

      # only ignore the outmost []
      next if ['[', ']'].include?(c) && brace == 0
      next if c == ',' && brace == 0

      buffer << c

      case c
      when '{' then brace += 1
      when '}'
        brace -= 1
        if brace == 0
          str = buffer.join
          @obj_callback.call(JSON.parse(str))
          buffer = []
        end
      end
    end
  rescue EOFError
  ensure
    @file.close
  end
end

out_path = "production_6d.csv"
File.open(out_path, "w") do |f|
  jr = JsonFileReader.new('production_6d.txt') do |o|
    quest = o['httpRequest']
    date = DateTime.strptime(o['timestamp'], "%Y-%m-%dT%T") #2020-03-29T00:46:14.702351978Z
    uri = URI(quest['requestUrl'])
    buf = [
      date.strftime("%FT%T"),
      quest['remoteIp'],
      uri.host,
      quest['requestMethod'],
      uri.path,
      quest['status'],
      quest['userAgent']&.sub(",", "."),
      quest['referer'],
      uri.query
    ].join(',') + "\n"

    f.write(buf)
  end
  jr.run
end

puts "fini: ", out_path