#!/usr/bin/env ruby ################## bin/mergelog -- brent@mbari.org ################## # Copyright (C) 2019 MBARI # MBARI Proprietary Information. All rights reserved. # # Merge ESP ASCII log with CSV log output from trimSTOQS utility # # Assumes that the first CSV column is timevalue # ####################################################################### require 'getoptlong' require 'timehash' require 'logtime' espOffset = csvOffset = Delay.new(0) #default to zero time offsets csvTZ = 'UTC' #lrauv is usually in UTC timezone ProgName = File.basename $0 $outSeparator = ' ' $tz = :dup writeESP = $stdout help = proc do STDERR.puts <<-END Merge ESP text log(s) with a CSV log -- 10/18/19 brent@mbari.org Typically reads CSV log text from stdin and writes ESP text log merged with CSV log to stdout. The first command argument is a , separated list of parameters to include. This list must be quoted if it contains any whitespace or is empty. Data fields/columns are output in the order listed. Each list element may be an abbreviation of the column name in the .csv header. If the --label option is specified, each field is prefixed with its column name or the specified abbreviation. The remaining command arguments are ESP text log files to read. Each successive log file specified must follow those before it in time. The first field of the CSV log (.csv) file must contain increasing timevalues Usually, the CSV log will have been output from the trimSTOQS utility. ESP timestamps are assumed to be localtime. CSV timestamps are assumed to be UTC. (unless --CSVzone specified) Usage: #{ProgName} {Options} columns,... {ESPlogText.out log(s)} Options: --ESPoffset=[0] #offset ESP time by given delay --CSVoffset=[0] #add specified offset to each timestamp read from CSV log --CSVzone=[#{csvTZ}] #default timezone for CSV log timestamps --CSVfile=path #read CSV data from file instead of stdin --epoch{=date} #output time as seconds from the given epoch --epoch{=offset} #begin epoch at first log time + offset seconds --labels #label output depth=xxx, foo=yyy instead of xxx, yyy --fullDate #include date in all timestamps --allTimes #timestamp every line output --tabs{=separator} #use TABS rather than spaces to separate output fields --writeESP=path #write ESP text log to given path instead of stdout --omitESP #suppress ESP text log output --local #output time in local time zone --utc #output time in UTC --debug #display debugging messages on stderr --help #displays this text Omit ESPlogText.out to read log, rather than CSV data, from stdin. Examples: #merge samples for depth, pitch and roll in erie.csv with ESP log real.out #{ProgName} real.out depth pitch roll argErr raise unless tz=argErr.message[/Unsupported Time Zone: \"(\w+)/,1] Time.setupZone tz $stderr.puts"Timezone changed to #{ParseDate::LocalTZ.join ','}" if $debug espTime = ParseDate.fromString part[1], espTime end newTime = espTime + espOffset if not timeString timeString = if $epoch epochOffset = if $epoch.empty? 0.0 else begin #assume relative to log start time if valid Float specified newTime.to_f - Float($epoch) rescue ArgumentError #otherwise, assume it's an epoch start date ParseDate.fromString($epoch<<"UTC").to_f end end proc do |t, ignored| "%.2f" % (t - epochOffset) end elsif $fullDate proc do |t, ignored| "@" << t.__send__($tz).asLogString(nil) end else lastTime={} proc do |t, outFile| zoneTxt = (zoneTime=t.__send__ $tz).asLogString lastTime[outFile] lastTime[outFile] = zoneTime "@" << zoneTxt end end end if csv unless csvTime #skip over context before ESP log began while csvLine = csv.gets csvColumn = csvLine.strip.split ',' csvTime = ParseDate.fromString(csvColumn.first+csvTZ, nil) + csvOffset break if csvTime >= espTime end end while csvTime <= espTime print timeString[csvTime, $stdout], $outSeparator outField=[] if remap.empty? if $label #all fields in header order csvColumn[1..-1].each_with_index do |data, index| outField << "#{csvTitle[index]}=#{data}" end else outField = csvColumn[1..-1] end else #remap/relabel fields if $label remap.each_with_index do |field, index| outField << "#{label[index]}=#{csvColumn[remap[index]+1]}" end else remap.each_with_index do |field, index| outField << csvColumn[remap[index]+1] end end end puts outField.join $outSeparator unless csvLine = csv.gets csv.close; csv = nil break end csvColumn = csvLine.strip.split ',' csvTime = ParseDate.fromString(csvColumn.first+csvTZ, nil) + csvOffset end end next unless writeESP lastTimeTxt = timeString[newTime, writeESP] espLine = lastTimeTxt << part[2] elsif lastTimeTxt and $allTimes writeESP.print lastTimeTxt, $outSeparator if writeESP end writeESP.puts espLine if writeESP end end