use 5.34.0; use Feature::Compat::Class; #============================================================================== class GeoStream; #============================================================================== use JSON::PP; use IO::Handle; use URI::Escape; use DBIx::Simple; use Data::Printer; use LWP::UserAgent; use Math::Trig qw/deg2rad/; use DateTime::Format::Strptime; use Term::ANSIColor qw/RESET :constants :constants256 color/; field $deduct_time :reader :param = 0; # time in seconds to deduct from log timestamps field $simulation :reader :param = 0; # if set, will simulate real-time delays based on log timestamps field $test_output :reader :param = 0; # if set will output all lines of log.txt field $sql_trace :reader :param = 0; # if set will output queries field $dbix = DBIx::Simple->connect( 'dbi:SQLite:dbname=locations.db' ); field $start; # only need this if simulating real-time delays ADJUST { # say $sql_trace; $dbix->dbh->trace('SQL|'.$sql_trace); # say "simulation: $simulation, deduct_time: $deduct_time"; $start = time - $deduct_time; # say "Start time: $start"; } my $ua = LWP::UserAgent->new( timeout => 10 ); my $json = JSON::PP->new->utf8; my $osm_url = 'https://nominatim.openstreetmap.org/reverse?format=jsonv2&lat=%s&lon=%s'; my @offsets = ([0.5,0.5],[0.25,0.25],[0.75,0.25],[0.25,0.75],[0.75,0.75]); # offsets to try if no data found my %seen; my $sim_startup_time; # time when simulation started method decode_line ($line) { say $line if $test_output; if ( my ($timestr) = $line =~ m{X-Plane Started on (.*)} ) { my $strp = DateTime::Format::Strptime->new( pattern => '%a %b %d %H:%M:%S %Y', time_zone => 'local', ); $sim_startup_time = $strp->parse_datetime($timestr); # $sim_startup_epoch = $dt->epoch; # warn "sim_startup_epoch: $sim_startup_epoch"; say 'Simulation startup time: ' . $sim_startup_time->strftime('%H:%M:%S'); # $sim_startup_time = time - $sim_startup_epoch - $deduct_time; } if ( my $data = $self->parse_log_line($line) ) { # aref [ $timestamp, $ms, $dir_name, $lat, $lon, $trisn ] # p $data; my ($timestamp, $ms, $dir_name, $lat, $lon, $tris) = @$data; # p $data; my ($h,$m,$s) = split /:/, $timestamp; my $secs = $h * 3600 + $m * 60 + $s; # say "$timestamp: $secs"; if ($simulation) { # only need this if simulating real-time delays # determine how long to sleep, to match log file timestamps my $target = $start + $secs; # warn $target; while (time < $target) { my $remaining = $target - time; say STDERR FAINT YELLOW, " waiting $remaining seconds ...", RESET; sleep 1; } } { my $load_time = $ms > 1_100_000 ? sprintf '%.1f sec', $ms / 1_000_000 : sprintf '%s ms', int $ms / 1_000; my $str = sprintf "%s [%s, %d triangles]", $dir_name, $load_time, $tris; say STDERR MAGENTA $str, RESET; } # calculate adjusted time string (start time + seconds from log line) my $hms = $sim_startup_time->clone->add(seconds => $secs) ->strftime('%H:%M:%S'); # warn "time: $hms"; my %res; my $id = join '~', $lat, $lon; # need to cache so not duplicating openstreetmap lookups if ( $seen{$id} ) { # p $seen{$id}; # next; $res{message} = "[$hms] loading from cache for $lat / $lon"; $res{colour} = 'ITALIC FAINT YELLOW'; # say STDERR ITALIC FAINT YELLOW # "[$timestamp] loading from cache for $lat / $lon", RESET; # p %seen; } elsif ( my $location = $self->db_lookup($lat, $lon) ) { $seen{$id}{data} = $location; $res{message} = "[$hms] loading from database for $lat / $lon"; $res{colour} = 'ITALIC GREEN'; # say STDERR ITALIC GREEN # "[$timestamp] loading from database for $lat / $lon", RESET; } else { $res{message} = "[$hms] fetching new location for $lat / $lon"; $res{colour} = 'ITALIC BLUE'; # say STDERR ITALIC FAINT BLUE # "[$timestamp] fetching new location for $lat / $lon", RESET; OFFSET: # try up to 5 times to get data, offset by 0.25-degree increments for my $o (@offsets) { my $offset_lat = $lat + $o->[0]; my $offset_lon = $lon + $o->[1]; my $url = sprintf( $osm_url, $offset_lat, $offset_lon ); my $res = $ua->get($url, 'User-Agent' => 'GeoStream/1.0'); if ($res->is_success) { my $data = eval { $json->decode($res->content) }; # p $data; next; # warn "no data for $offset_lat / $offset_lon, trying next" and next OFFSET if $data->{error}; # say $dir_name; if ( my $addr = $data->{address} ) { my $location = $self->format_address($addr); $seen{$id}{data} = $location; last OFFSET; } } else { warn "HTTP error: " . $res->status_line . "\n"; } } $seen{$id}{data} ||= 'No data'; # couldn't find anything in osm resoiurce } my $dimensions = $self->tile_dimensions($lat); $res{message} .= sprintf " :: Tile area: %s [H-%s x W-%s] km^2", map int $_, @{$dimensions}{qw/area height width/}; say color($res{colour}), ' ', $res{message}, RESET, ' :: ', CYAN, $seen{$id}{data}, RESET; # log message if it contains 'unknown' or 'no data': if ( $seen{$id}{data} =~ /no data/i ) { open my $log_fh, '>>', 'geo_stream_warnings.log'; say $log_fh join(' :: ', $lat, $lon, $seen{$id}{data} ); close $log_fh; } } } method format_address ($addr) { # Step 1: Pick first meaningful locality (city, town, village, municipality) my @first_candidates = qw(city town village municipality city_district); =begin my ($first) = grep { defined $_ && $_ ne '' && $_ !~ /^unknown$/i } map { $addr->{$_} } @first_candidates; =cut my $first; for (@first_candidates) { $first = $self->clean_address($addr->{$_}); last if $first; } # Step 2: Pick the most useful middle field my @middle_candidates = qw(state county region state_district province); my $middle; for (@middle_candidates) { $middle = $self->clean_address($addr->{$_}); last if $middle && $middle ne $first; # avoid duplicate } # Step 3: Always pick country my $country = $self->clean_address($addr->{country}); =begin my ($city) = grep { defined $_ } # get first one of: @$addr{ qw/city city_district town village/ }; # || next OFFSET; my $region = $addr->{county} || $addr->{state}; my $country = $addr->{country}; $seen{$id}{data} = join ', ', map { $_ || 'Unknown' } $city, $region, $country; =cut # Build final parts my @parts = grep { defined $_ && $_ ne '' } ($first, $middle, $country); # Join non-empty parts with comma my $formatted = join ', ', @parts; return $formatted; } method clean_address ($place) { defined $place && $place !~ /^unknown$/i ? $place : (); } method parse_log_line { #0:57:32.110 I/SCN: DSF load time: 5710065 for file D:\XP Map Enhacement\Base packages\XPME_Europe/Earth nav data/+50+000/+53+007.dsf (608040 tris, 134 skipped for -23.8 m^2) my $line = shift; if ( $line =~ m{ (\d+:\d+:\d+) # [1] timestamp (e.g. 0:57:32) \.\d+ # fractional seconds (ignored) \s+I\/SCN:\s+ # literal " I/SCN: " with spacing DSF\ load\ time:\s+ # literal "DSF load time:" (\d+).* # [2] load time in microseconds [/\\]([^/\\]+) # [3] dir_name (after last / or \) /Earth\ nav\ data/ # folder [+-]\d+[+-]\d+/ # folder with lat/lon ([+-]\d+) # [4] latitude ([+-]\d+) # [5] longitude \.dsf # file extension \s+\( # opening parenthesis (\d+) # [6] triangle count \s+tris,\s+\d+\s+skipped\s+for\s+ (-?[\d\.]+)\s+m\^2\) # [7] area value of skipped triangles (can be negative) }x ) { my ($time, $ms, $dir_name, $lat, $lon, $tris) = ($1, $2, $3, $4, $5, $6); # say join ' :: ', $time, $ms, $dir_name, $lat, $lon, $tris; return [ $time, $ms, $dir_name, $lat, $lon, $tris ]; } return undef; } method tile_dimensions { my $lat = shift; my $height = 111.32; # km per degree latitude my $width = 111.32 * cos(deg2rad($lat)); my $area = $width * $height; return { width => $width, height => $height, area => $area }; } method db_lookup ($lat, $lon) { my %h = ( lat => int $lat, lon => int $lon ); # convert to integers eg +53 $dbix->select('locations', 'name', \%h )->into(my $location); return $location; } method get_sql_trace_level { $sql_trace } 1;