use 5.34.0;
use Feature::Compat::Class;
#==============================================================================
class GeoStream;
#==============================================================================
use JSON::PP;
use IO::Handle;
use URI::Escape;
use DBIx::Simple;
use Data::Printer;
use LWP::UserAgent;
use Math::Trig qw/deg2rad/;
use Term::ANSIColor qw/RESET :constants :constants256 color/;
field $deduct_time :reader :param = 0; # time in seconds to deduct from log timestamps
field $simulation :reader :param = 0; # if set, will simulate real-time delays based on log timestamps
field $test_output :reader :param = 0; # if set will output all lines of log.txt
field $sql_trace :reader :param = 0; # if set will output queries
field $dbix = DBIx::Simple->connect( 'dbi:SQLite:dbname=locations.db' );
# only need this if simulating real-time delays
field $start;
ADJUST { say $sql_trace;
$dbix->dbh->trace('SQL|'.$sql_trace);
# say "simulation: $simulation, deduct_time: $deduct_time";
$start = time - $deduct_time; # say "Start time: $start";
}
my $ua = LWP::UserAgent->new( timeout => 10 );
my $json = JSON::PP->new->utf8;
my $osm_url = 'https://nominatim.openstreetmap.org/reverse?format=jsonv2&lat=%s&lon=%s';
my @offsets = ([0.5,0.5],[0.25,0.25],[0.75,0.25],[0.25,0.75],[0.75,0.75]); # offsets to try if no data found
my %seen;
method decode_line ($line) { say $line if $test_output;
if ( my $data = $self->parse_log_line($line) ) { # aref [ $timestamp, $ms, $dir_name, $lat, $lon, $trisn ]
# p $data;
my ($timestamp, $ms, $dir_name, $lat, $lon, $tris) = @$data; # p $data;
my ($h,$m,$s) = split /:/, $timestamp;
my $secs = $h * 3600 + $m * 60 + $s; # say "$timestamp: $secs";
if ($simulation) { # only need this if simulating real-time delays
# determine how long to sleep, to match log file timestamps
my $target = $start + $secs; # warn $target;
while (time < $target) {
my $remaining = $target - time;
say STDERR FAINT YELLOW, " waiting $remaining seconds ...",
RESET;
sleep 1;
}
}
my $str = sprintf "%s [%s ms, %d triangles]",
$dir_name, int $ms / 1000, $tris;
say STDERR MAGENTA $str, RESET;
my %res;
my $id = join '~', $lat, $lon;
# need to cache so not duplicating openstreetmap lookups
if ( $seen{$id} ) { # p $seen{$id}; # next;
$res{message} = "[$timestamp] loading from cache for $lat / $lon";
$res{colour} = 'ITALIC FAINT YELLOW';
# say STDERR ITALIC FAINT YELLOW
# "[$timestamp] loading from cache for $lat / $lon", RESET; # p %seen;
}
elsif ( my $location = $self->db_lookup($lat, $lon) ) {
$seen{$id}{data} = $location;
$res{message} = "[$timestamp] loading from database for $lat / $lon";
$res{colour} = 'ITALIC GREEN';
# say STDERR ITALIC GREEN
# "[$timestamp] loading from database for $lat / $lon", RESET;
}
else {
$res{message} = "[$timestamp] fetching new location for $lat / $lon";
$res{colour} = 'ITALIC BLUE';
# say STDERR ITALIC FAINT BLUE
# "[$timestamp] fetching new location for $lat / $lon", RESET;
OFFSET: # try up to 5 times to get data, offset by 0.25-degree increments
for my $o (@offsets) {
my $offset_lat = $lat + $o->[0];
my $offset_lon = $lon + $o->[1];
my $url = sprintf( $osm_url, $offset_lat, $offset_lon );
my $res = $ua->get($url, 'User-Agent' => 'GeoStream/1.0');
if ($res->is_success) {
my $data = eval { $json->decode($res->content) }; # p $data; next;
# warn "no data for $offset_lat / $offset_lon, trying next" and
next OFFSET if $data->{error};
# say $dir_name;
if ( my $addr = $data->{address} ) {
my ($city) = grep { defined $_ } # get first one of:
@$addr{ qw/city city_district town village/ };
# || next OFFSET;
my $region = $addr->{county} || $addr->{state};
my $country = $addr->{country};
$seen{$id}{data} = join ', ', map { $_ || 'Unknown' }
$city, $region, $country;
# say "\tCoordinates: $lat,$lon => $city, $region, $country";
last OFFSET;
}
}
else {
warn "HTTP error: " . $res->status_line . "\n";
}
}
$seen{$id}{data} ||= 'No data'; # couldn't find anything in osm resoiurce
}
my $dimensions = $self->tile_dimensions($lat);
$res{message} .= sprintf " :: Tile area: %s [H-%s x W-%s] km^2",
map int $_, @{$dimensions}{qw/area height width/};
say color($res{colour}), ' ', $res{message}, RESET, ' :: ',
CYAN, $seen{$id}{data}, RESET;
# log message if it contains 'unknown' or 'no data':
if ( $seen{$id}{data} =~ /unknown|no data/i ) {
open my $log_fh, '>>', 'geo_stream_warnings.log';
say $log_fh join(' :: ', $lat, $lon, $seen{$id}{data} );
close $log_fh;
}
}
}
method parse_log_line {
#0:57:32.110 I/SCN: DSF load time: 5710065 for file D:\XP Map Enhacement\Base packages\XPME_Europe/Earth nav data/+50+000/+53+007.dsf (608040 tris, 134 skipped for -23.8 m^2)
my $line = shift;
if ( $line =~ m{
(\d+:\d+:\d+) # [1] timestamp (e.g. 0:57:32)
\.\d+ # fractional seconds (ignored)
\s+I\/SCN:\s+ # literal " I/SCN: " with spacing
DSF\ load\ time:\s+ # literal "DSF load time:"
(\d+).* # [2] load time in microseconds
[/\\]([^/\\]+) # [3] dir_name (after last / or \)
/Earth\ nav\ data/ # folder
[+-]\d+[+-]\d+/ # folder with lat/lon
([+-]\d+) # [4] latitude
([+-]\d+) # [5] longitude
\.dsf # file extension
\s+\( # opening parenthesis
(\d+) # [6] triangle count
\s+tris,\s+\d+\s+skipped\s+for\s+
(-?[\d\.]+)\s+m\^2\) # [7] area value of skipped triangles (can be negative)
}x ) {
my ($time, $ms, $dir_name, $lat, $lon, $tris) = ($1, $2, $3, $4, $5, $6);
# say join ' :: ', $time, $ms, $dir_name, $lat, $lon, $tris;
return [ $time, $ms, $dir_name, $lat, $lon, $tris ];
}
return undef;
}
method tile_dimensions {
my $lat = shift;
my $height = 111.32; # km per degree latitude
my $width = 111.32 * cos(deg2rad($lat));
my $area = $width * $height;
return { width => $width, height => $height, area => $area };
}
method db_lookup ($lat, $lon) {
my %h = ( lat => int $lat, lon => int $lon ); # convert to integers eg +53
$dbix->select('locations', 'name', \%h )->into(my $location);
return $location;
}
method get_sql_trace_level { $sql_trace }
1;
