#!/usr/bin/env ruby
##################  bin/mergelog -- brent@mbari.org  ##################
#    Copyright (C) 2019 MBARI
#    MBARI Proprietary Information. All rights reserved.
#
# Merge ESP ASCII log with CSV log output from trimSTOQS utility
#
# Assumes that the first CSV column is timevalue
#
#######################################################################

require 'getoptlong'
require 'timehash'
require 'logtime'

espOffset = csvOffset = Delay.new(0)   #default to zero time offsets
csvTZ = 'UTC'  #lrauv is usually in UTC timezone

ProgName = File.basename $0
$outSeparator = ' '
$tz = :dup
writeESP = $stdout

help = proc do
      STDERR.puts <<-END
Merge ESP text log(s) with a CSV log -- 10/18/19 brent@mbari.org

Typically reads CSV log text from stdin and writes ESP text log
merged with CSV log to stdout.

The first command argument is a , separated list of parameters to include.
This list must be quoted if it contains any whitespace or is empty.
Data fields/columns are output in the order listed.
Each list element may be an abbreviation of the column name
in the .csv header.

If the --label option is specified,
each field is prefixed with its column name or the specified abbreviation.

The remaining command arguments are ESP text log files to read.
Each successive log file specified must follow those before it in time.

The first field of the CSV log (.csv) file must contain increasing timevalues
Usually, the CSV log will have been output from the trimSTOQS utility.
ESP timestamps are assumed to be localtime.
CSV timestamps are assumed to be UTC. (unless --CSVzone specified)

Usage:
  #{ProgName} {Options} columns,... {ESPlogText.out log(s)}
Options:
  --ESPoffset=[0]     #offset ESP time by given delay <hh:mm:ss.fraction>
  --CSVoffset=[0]     #add specified offset to each timestamp read from CSV log
  --CSVzone=[#{csvTZ}]     #default timezone for CSV log timestamps
  --CSVfile=path      #read CSV data from file instead of stdin
  --epoch{=date}      #output time as seconds from the given epoch
  --epoch{=offset}    #begin epoch at first log time + offset seconds
  --labels            #label output depth=xxx, foo=yyy instead of xxx, yyy
  --fullDate          #include date in all timestamps
  --allTimes          #timestamp every line output
  --tabs{=separator}  #use TABS rather than spaces to separate output fields
  --writeESP=path     #write ESP text log to given path instead of stdout
  --omitESP           #suppress ESP text log output
  --local             #output time in local time zone
  --utc               #output time in UTC
  --debug             #display debugging messages on stderr
  --help              #displays this text

Omit ESPlogText.out to read log, rather than CSV data, from stdin.

Examples:
#merge samples for depth, pitch and roll in erie.csv with ESP log real.out
  #{ProgName} real.out depth pitch roll <erie.csv
#the same using compressed csv data
  zcat erie.csv | #{ProgName} real.out depth pitch roll
#convert times from any local timezone to HST
  #{ProgName} --utc ESPlogText.out </dev/null | TZ=HST mergelog --local
      END
    exit 2
end

opts = GetoptLong.new(
  ["--ESPoffset", '-E', GetoptLong::REQUIRED_ARGUMENT],
  ["--CSVoffset", '-C', GetoptLong::REQUIRED_ARGUMENT],
  ["--CSVzone", '-z',   GetoptLong::REQUIRED_ARGUMENT],
  ["--CSVfile", '-F',   GetoptLong::REQUIRED_ARGUMENT],
  ["--epoch",'-e',      GetoptLong::OPTIONAL_ARGUMENT],
  ["--writeESP",'-w',   GetoptLong::REQUIRED_ARGUMENT],
  ["--omitESP", '-o',   GetoptLong::NO_ARGUMENT],
  ["--labels", '-l',    GetoptLong::NO_ARGUMENT],
  ["--fullDate", '-f',  GetoptLong::NO_ARGUMENT],
  ["--allTimes", '-a',  GetoptLong::NO_ARGUMENT],
  ["--tabs", '-t',      GetoptLong::OPTIONAL_ARGUMENT],
  ["--utc", '-U',       GetoptLong::NO_ARGUMENT],
  ["--local", '-L',     GetoptLong::NO_ARGUMENT],
  ["--debug", '-d',     GetoptLong::NO_ARGUMENT],
  ["--help", "-h",      GetoptLong::NO_ARGUMENT]
)

csv = $stdin
opts.each do |opt, arg|
  case opt
    when "--ESPoffset"
      espOffset = Delay.parse arg
    when "--CSVoffset"
      csvOffset = Delay.parse arg
    when "--CSVzone"
      csvTZ = arg
    when "--CSVfile"
      csv = open arg, 'r'
    when "--labels"
      $label = true
    when "--fullDate"
      $fullDate = true
    when "--epoch"
      $epoch = arg
    when "--writeESP"
      writeESP = arg.empty? ? nil : open(arg, 'w');
    when "--omitESP"
      writeESP = nil
    when "--allTimes"
      $allTimes = true
    when "--tabs"
      $outSeparator = arg.empty? ? "\t" : arg
    when "--utc"
      $tz = :getutc
    when "--local"
      $tz = :getlocal
    when "--debug"
      $debug = true
    else
      help.call
  end
end

label = ARGV.shift.split(',')   #column labels
case ARGV  #no csv if csvFile not specified and reading log from stdin
  when nil, '', '-'
    csv = nil if csv == $stdin
    ARGV.unshift ''   #read log from stdin if no file spec'd
end

if csv and csvLine = csv.gets
  csvTitle = csvLine.strip.split ','
  firstValue = csvTitle.shift
  raise "First column must be timevalue, not #{firstValue}" unless
    firstValue == "timevalue"

  remap = []
  for prefix in label
    index = csvTitle.index{|fieldName| fieldName[0..prefix.length-1] == prefix}
    raise "No field's name begins with #{prefix}" unless index
    remap << index
  end

  csvHeader=['time']
  if remap.empty?
    csvHeader=csvHeader.concat csvTitle
  else
    remap.each_with_index do |field, index|
      csvHeader << csvTitle[remap[index]]
    end
  end
else  #there is no CSV data
  csv = nil
end

espTimestamp = /\A@(\S*)(.*)/  #separate timestamp from log line
lastTimeTxt = csvColumn = timeString = nil

while espFn = ARGV.shift
  esp = ['','-'].include?(espFn) ? $stdin : open(espFn)
  if csv
    title=csvHeader
    title=title.dup<<File.expand_path(espFn) unless esp==$stdin
    #append the data file name to output title
    print '#', title.join($outSeparator), "\n"
    csvTime = nil
  end
  espTime = nil  #first timestamp must include year

  while espLine=esp.gets
    espLine.strip!
    if part = espTimestamp.match(espLine)
      begin
        espTime = ParseDate.fromString part[1], espTime
      rescue ArgumentError=>argErr
        raise unless tz=argErr.message[/Unsupported Time Zone: \"(\w+)/,1]
        Time.setupZone tz
        $stderr.puts"Timezone changed to #{ParseDate::LocalTZ.join ','}" if $debug
        espTime = ParseDate.fromString part[1], espTime
      end
      newTime = espTime + espOffset
      if not timeString
        timeString = if $epoch
          epochOffset = if $epoch.empty?
            0.0
          else
            begin  #assume relative to log start time if valid Float specified
              newTime.to_f - Float($epoch)
            rescue ArgumentError  #otherwise, assume it's an epoch start date
              ParseDate.fromString($epoch<<"UTC").to_f
            end
          end
          proc do |t, ignored|
            "%.2f" % (t - epochOffset)
          end
        elsif $fullDate
          proc do |t, ignored|
            "@" << t.__send__($tz).asLogString(nil)
          end
        else
          lastTime={}
          proc do |t, outFile|
            zoneTxt = (zoneTime=t.__send__ $tz).asLogString lastTime[outFile]
            lastTime[outFile] = zoneTime
            "@" << zoneTxt
          end
        end
      end
      if csv
        unless csvTime  #skip over context before ESP log began
          while csvLine = csv.gets
            csvColumn = csvLine.strip.split ','
            csvTime = ParseDate.fromString(csvColumn.first+csvTZ, nil) + csvOffset
            break if csvTime >= espTime
          end
        end
        while csvTime <= espTime
          print timeString[csvTime, $stdout], $outSeparator
          outField=[]
          if remap.empty?
            if $label  #all fields in header order
              csvColumn[1..-1].each_with_index do |data, index|
                outField << "#{csvTitle[index]}=#{data}"
              end
            else
              outField = csvColumn[1..-1]
            end
          else  #remap/relabel fields
            if $label
              remap.each_with_index do |field, index|
                outField << "#{label[index]}=#{csvColumn[remap[index]+1]}"
              end
            else
              remap.each_with_index do |field, index|
                outField << csvColumn[remap[index]+1]
              end
            end
          end
          puts outField.join $outSeparator
          unless csvLine = csv.gets
            csv.close; csv = nil
            break
          end
          csvColumn = csvLine.strip.split ','
          csvTime = ParseDate.fromString(csvColumn.first+csvTZ, nil) + csvOffset
        end
      end
      next unless writeESP
      lastTimeTxt = timeString[newTime, writeESP]
      espLine = lastTimeTxt << part[2]
    elsif lastTimeTxt and $allTimes
      writeESP.print lastTimeTxt, $outSeparator if writeESP
    end
    writeESP.puts espLine if writeESP
  end

end